mirror of
https://bitbucket.org/jsuto/piler.git
synced 2025-04-07 02:50:32 +02:00
Compare commits
No commits in common. "master" and "v0.1.23" have entirely different histories.
26
.gitignore
vendored
26
.gitignore
vendored
@ -1,26 +0,0 @@
|
||||
Makefile
|
||||
*.o
|
||||
*.so
|
||||
*.a
|
||||
*.deb
|
||||
src/libpiler.so.0*
|
||||
src/piler
|
||||
src/pilerconf
|
||||
src/piler-smtp
|
||||
src/pileraget
|
||||
src/pilerexport
|
||||
src/pilerget
|
||||
src/pilerimport
|
||||
src/pilerstats
|
||||
src/pilertest
|
||||
src/reindex
|
||||
unit_tests/check_attachments
|
||||
unit_tests/check_decoder
|
||||
unit_tests/check_digest
|
||||
unit_tests/check_hash
|
||||
unit_tests/check_misc
|
||||
unit_tests/check_mydomains
|
||||
unit_tests/check_parser
|
||||
unit_tests/check_parser_utils
|
||||
unit_tests/check_rules
|
||||
unit_tests/smtp
|
||||
3
CREDITS
3
CREDITS
@ -4,5 +4,4 @@ The FSF.hu Foundation (http://fsf.hu/) supported and donated piler within the
|
||||
Nemeth Adam reviewed the web interface, and gave lots of useful hints and insights to
|
||||
improve the web ui of piler.
|
||||
|
||||
Remi Smith improved the restricted auditor feature for a better multitenancy,
|
||||
and invented the default theme.
|
||||
Remi Smith improved the restricted auditor feature for a better multitenancy.
|
||||
|
||||
12
G
12
G
@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o nounset
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
read -r version < VERSION
|
||||
|
||||
tarball="piler-${version}.tar.gz"
|
||||
|
||||
git archive --prefix="piler-${version}/" HEAD | gzip -c > "$tarball"
|
||||
sha256sum "$tarball"
|
||||
3
LICENSE
3
LICENSE
@ -1,6 +1,6 @@
|
||||
piler, an enterprise level email archiving application
|
||||
|
||||
Copyright (C) 2012-2023, Janos SUTO <sj@acts.hu>
|
||||
Copyright (C) 2012-2013, Janos SUTO <sj@acts.hu>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
@ -13,3 +13,4 @@ GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
32
Makefile.in
32
Makefile.in
@ -16,9 +16,9 @@ localstatedir = @localstatedir@
|
||||
CC = @CC@
|
||||
CFLAGS = @CFLAGS@ @CPPFLAGS@
|
||||
DEFS = @defs@
|
||||
INCDIR = -I. @INCDIR@ @sql_includes@
|
||||
INCDIR = -I. @INCDIR@ @mysql_includes@
|
||||
LIBDIR = -L. @LIBDIR@ @LDFLAGS@
|
||||
LIBS = @LIBS@ @sql_libs@
|
||||
LIBS = @LIBS@ @mysql_libs@
|
||||
RUNNING_USER = @RUNNING_USER@
|
||||
RUNNING_GROUP = `@id_bin@ -gn $(RUNNING_USER)`
|
||||
|
||||
@ -33,7 +33,7 @@ INSTALL = @INSTALL@
|
||||
am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
|
||||
piler-config.h stamp.h stamp-h1 params.h
|
||||
|
||||
all: all-recursive config-php
|
||||
all: all-recursive
|
||||
install: installdirs install-recursive
|
||||
|
||||
|
||||
@ -63,36 +63,20 @@ $(RECURSIVE_TARGETS):
|
||||
fi; test -z "$$fail"
|
||||
|
||||
|
||||
config-php:
|
||||
sed -e "s%SYSCONFDIR%$(sysconfdir)%g" -e "s%SBINDIR%$(sbindir)%g" config.php.in > webui/config.php
|
||||
sed -e "s%BINDIR%$(bindir)%g" -i webui/config.php
|
||||
|
||||
|
||||
installdirs: mkinstalldirs
|
||||
$(srcdir)/mkinstalldirs \
|
||||
$(DESTDIR)$(bindir) \
|
||||
$(DESTDIR)$(sbindir) \
|
||||
$(DESTDIR)$(libdir) \
|
||||
$(DESTDIR)$(libexecdir)/piler \
|
||||
$(DESTDIR)$(datarootdir)/piler \
|
||||
$(DESTDIR)$(sysconfdir) \
|
||||
$(DESTDIR)$(sysconfdir)/piler \
|
||||
$(DESTDIR)/etc/init.d \
|
||||
$(DESTDIR)$(bindir) $(DESTDIR)$(sbindir) $(DESTDIR)$(libdir) $(DESTDIR)$(libexecdir)/piler $(DESTDIR)$(sysconfdir) \
|
||||
$(DESTDIR)$(localstatedir)/piler/store \
|
||||
$(DESTDIR)$(localstatedir)/piler/stat \
|
||||
$(DESTDIR)$(localstatedir)/piler/tmp \
|
||||
$(DESTDIR)$(localstatedir)/piler/error \
|
||||
$(DESTDIR)$(localstatedir)/piler/export \
|
||||
$(DESTDIR)$(localstatedir)/piler/manticore
|
||||
$(DESTDIR)$(localstatedir)/piler/stat $(DESTDIR)$(localstatedir)/piler/tmp \
|
||||
$(DESTDIR)$(localstatedir)/piler/sphinx
|
||||
|
||||
$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
|
||||
$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store
|
||||
$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/imap
|
||||
$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
|
||||
$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
|
||||
$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
|
||||
$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
|
||||
$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore
|
||||
$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx
|
||||
|
||||
|
||||
install-am:
|
||||
@ -126,7 +110,7 @@ key:
|
||||
dd if=/dev/urandom bs=56 count=1 of=piler.key
|
||||
|
||||
postinstall:
|
||||
@bash util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)
|
||||
@sh util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir)
|
||||
|
||||
|
||||
|
||||
|
||||
6
README
6
README
@ -24,9 +24,9 @@ Features:
|
||||
|
||||
- AD / LDAP authentication
|
||||
|
||||
- IMAP, POP3 authentication
|
||||
- IMAP authentication
|
||||
|
||||
- single sign-on (SSO)
|
||||
- single sign-on
|
||||
|
||||
- Google Apps support
|
||||
|
||||
@ -34,8 +34,6 @@ Features:
|
||||
|
||||
- STARTTLS support
|
||||
|
||||
- Google Authenticator support for 2-factor authentication
|
||||
|
||||
- i18n
|
||||
- customisable theme
|
||||
|
||||
|
||||
209
RELEASE_NOTES
209
RELEASE_NOTES
@ -1,209 +0,0 @@
|
||||
1.4.5:
|
||||
------
|
||||
|
||||
- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
|
||||
|
||||
; max message size in bytes
|
||||
; piler-smtp will reject any message that's bigger than this number
|
||||
max_message_size=50000000
|
||||
|
||||
; max memory in bytes piler-smtp uses for buffering messages
|
||||
; when this limit is exceeded, no new emails will be accepted
|
||||
; until the used memory for all in progress emails decreases
|
||||
; below this level
|
||||
max_smtp_memory=500000000
|
||||
|
||||
Be sure to adjust these values to your environment!
|
||||
|
||||
- Added read-only connection support for manticore
|
||||
If using sphinx, add the following to config-site.php:
|
||||
$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
|
||||
|
||||
- pilerimport supports Zimbra IMAP impersonation
|
||||
|
||||
Generate the following base64 encoded string:
|
||||
(Be sure to use the actual usernames and password):
|
||||
|
||||
pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
|
||||
|
||||
Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
|
||||
|
||||
pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
|
||||
|
||||
Note that "ZIMBRA" is a special username, it indicates for pilerimport
|
||||
to actually use the imap impersonation for Zimbra.
|
||||
|
||||
|
||||
1.4.4:
|
||||
------
|
||||
|
||||
- Renamed "group" table to "usergroup"
|
||||
Be sure to run util/db-upgrade.sql on the mysql piler database
|
||||
|
||||
|
||||
1.4.3:
|
||||
------
|
||||
|
||||
- Improved real-time index support (no need to use rtindex.py any more)
|
||||
See https://www.mailpiler.org/wiki/current:manticore for more
|
||||
|
||||
|
||||
1.4.2:
|
||||
------
|
||||
|
||||
- Fixed docker image
|
||||
- Fixed php socket path in piler-nginx.conf
|
||||
- Fixed manticore index data settings
|
||||
|
||||
|
||||
1.4.1:
|
||||
------
|
||||
|
||||
- Replaced sphinx with manticore 5.0.2, see https://www.mailpiler.org/wiki/current:manticore
|
||||
- Replaced obsoleted libchart library with chart.js
|
||||
- Added real-time index support
|
||||
- Added oauth2 support to imapfetch.py
|
||||
- Added support for pilerexport to spread files among several directories. See the -D option
|
||||
|
||||
|
||||
1.3.12:
|
||||
-------
|
||||
|
||||
- Introduced new piler.conf variable: tls_min_version
|
||||
|
||||
It sets the minimum TLS protocol version the piler-smtp daemon supports.
|
||||
|
||||
Possible values:
|
||||
- TLSv1 (not recommended)
|
||||
- TLSv1.1 (not recommended)
|
||||
- TLSv1.2 (default)
|
||||
- TLSv1.3
|
||||
|
||||
- Introduced the archive_address feature, see etc/example.conf for the details
|
||||
- Introduced the raw: search label
|
||||
- Added Italian translation. Credits: Stefano Gatto
|
||||
- timestamp signing sorts by 'id' column
|
||||
- timestamp hash value defaults to sha256
|
||||
- Minor fixes
|
||||
- Added support for php 8.1
|
||||
- Fixed handling long email addresses
|
||||
|
||||
Be sure to apply util/db-upgrade.sql
|
||||
|
||||
|
||||
1.3.11:
|
||||
-------
|
||||
|
||||
- [BUGFIX] Refactored the smtp timeout check
|
||||
- Obsoleted the LDAP port parameter. Specify the ldap host in the form
|
||||
of protocol://hostname:port, eg. ldaps://ldap.example.com:636
|
||||
|
||||
1.3.10:
|
||||
-------
|
||||
|
||||
- Added security header feature
|
||||
- Introduced the smtp acl list, and obsoleted the tcp_wrappers check
|
||||
- Switched from Blowfish encryption to AES-256
|
||||
|
||||
1.3.9:
|
||||
------
|
||||
|
||||
- Added a separator to searching for attachment names
|
||||
- [BUGFIX] Render multiple mail parts in mail view instead of only the last part
|
||||
- Use TLS v1.2 with openssl 1.0.x for connecting remote pop3/imap servers
|
||||
- Instant search results to the gui when the search page loads
|
||||
- Support sphinx-3.3.1, introduced sphinx strict mode variable
|
||||
- GUI domain fixes
|
||||
- gcc 9 fixes
|
||||
- Fix permission on sphinx data dir to 700
|
||||
- pilerpurge.py should honor the mysqlhost value
|
||||
- Password change enabled by default
|
||||
- Health page fixes
|
||||
- GUI mime parser fixes
|
||||
- Start/stop script fix
|
||||
- Optimized search page for mobile devices, set $config['ENABLE_MOBILE_PREVIEW'] = 1; in config-site.php to enable it
|
||||
|
||||
|
||||
1.3.0:
|
||||
-----
|
||||
|
||||
- Switched from utf8 to utf8mb4 encoding.
|
||||
|
||||
- Switched from preforking model to epoll
|
||||
|
||||
- Removed PDF support
|
||||
|
||||
- GUI fixes
|
||||
|
||||
- Added HTML purifier support
|
||||
|
||||
- Rewritten pilerpurge in Python
|
||||
|
||||
1.2.0:
|
||||
------
|
||||
|
||||
Heavy SQL scheme changes throughout the development of 1.2.0!
|
||||
|
||||
2016.10.06.
|
||||
|
||||
Removed deb and rpm packaging metadata.
|
||||
|
||||
2016.09.21.
|
||||
|
||||
Improved logging for the GUI. See the new config option: $config['LOG_LEVEL'] = NORMAL;
|
||||
|
||||
2016.09.18.
|
||||
|
||||
Moved all piler configuration from ${sysconfdir} to ${sysconfdir}/piler, eg.
|
||||
|
||||
root@01b74b787c26:/# ls -la /usr/local/etc/piler/
|
||||
total 36
|
||||
drwxr-xr-x 1 root root 24 Oct 6 20:49 .
|
||||
drwxr-xr-x 1 root root 74 Oct 6 20:49 ..
|
||||
-rw-r--r-- 4 root root 1008 Oct 6 20:39 piler.conf
|
||||
-rw-r----- 5 root piler 1066 Oct 6 20:39 piler.conf.dist
|
||||
-rw-r----- 5 root piler 56 Oct 6 20:40 piler.key
|
||||
-rw------- 5 root root 5256 Oct 6 20:40 piler.pem
|
||||
-rw-r--r-- 1 root root 6270 Oct 6 20:49 sphinx.conf
|
||||
-rw-r--r-- 5 root piler 6299 Oct 6 20:39 sphinx.conf.dist
|
||||
|
||||
Make sure to update the piler shipped shell scripts
|
||||
(rc.searchd, indexer.*.sh, ...) as well!
|
||||
|
||||
2016.08.31.
|
||||
|
||||
Added chunking / BDAT support for the piler daemon.
|
||||
You may turn it on by setting enable_chunking=1 in piler.conf
|
||||
|
||||
2016.06.19.
|
||||
|
||||
Improved attachment table indexing.
|
||||
|
||||
2016.05.07.
|
||||
|
||||
Obsoleted the --enable-starttls configure option.
|
||||
|
||||
2016.03.02.
|
||||
|
||||
Discard messages shorther than 100 bytes.
|
||||
|
||||
2016.02.10.
|
||||
|
||||
Added the private feature (for auditors only).
|
||||
|
||||
|
||||
2015.11.26.
|
||||
|
||||
Removed the pilergetd stuff.
|
||||
|
||||
2015.07.08.
|
||||
|
||||
Delete option for auditors.
|
||||
|
||||
2015.05.07.
|
||||
|
||||
Added milter script to contrib directory.
|
||||
|
||||
2015.04.22.
|
||||
|
||||
Added timestamp support.
|
||||
20
SECURITY.md
20
SECURITY.md
@ -1,20 +0,0 @@
|
||||
Disclosure policy
|
||||
|
||||
If you find a security issue, please contact the project owner at sj@acts.hu
|
||||
with the details (ie. piler version, details of the setup, how to exploit the
|
||||
vulnerability, etc).
|
||||
|
||||
Please provide 30 days for verifying the vulnerability, fixing the issue, and
|
||||
notifying the piler users.
|
||||
|
||||
Security update policy
|
||||
|
||||
If a security vulnerability has found, the details, possible mitigations,
|
||||
workarounds, etc. will be shared on the piler mailing list (piler-user@list.acts.hu)
|
||||
and on the wiki: https://www.mailpiler.org/
|
||||
|
||||
Security configurations
|
||||
|
||||
- Use https for the GUI
|
||||
- Reset the default passwords for admin and auditor
|
||||
- Use the smtp acl feature to restrict SMTP access to the archive, see https://mailpiler.com/smtp-acl-list/
|
||||
4
aclocal.m4
vendored
4
aclocal.m4
vendored
@ -13,8 +13,8 @@
|
||||
|
||||
m4_ifndef([AC_AUTOCONF_VERSION],
|
||||
[m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
|
||||
m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.69],,
|
||||
[m4_warning([this file was generated for autoconf 2.69.
|
||||
m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.68],,
|
||||
[m4_warning([this file was generated for autoconf 2.68.
|
||||
You have another version of autoconf. It may work, but is not guaranteed to.
|
||||
If you have problems, you may need to regenerate the build system entirely.
|
||||
To do so, use the procedure documented by the package, typically `autoreconf'.])])
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
# This is a sample build configuration for all languages.
|
||||
# Check our guides at https://confluence.atlassian.com/x/VYk8Lw for more examples.
|
||||
# Only use spaces to indent your .yml configuration.
|
||||
# -----
|
||||
# You can specify a custom docker image from Docker Hub as your build environment.
|
||||
|
||||
image: sutoj/builder:bionic
|
||||
clone:
|
||||
depth: 5
|
||||
|
||||
pipelines:
|
||||
default:
|
||||
- step:
|
||||
script:
|
||||
- service mysql start
|
||||
- ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb
|
||||
- make clean all install
|
||||
- mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql
|
||||
- cd unit_tests
|
||||
- ./run.sh
|
||||
- cd ../webui
|
||||
- phpunit
|
||||
562
config.php.in
562
config.php.in
@ -1,562 +0,0 @@
|
||||
<?php
|
||||
|
||||
ini_set('session.cookie_httponly', true);
|
||||
ini_set('session.use_strict_mode', 1);
|
||||
ini_set('session.use_only_cookies', 1);
|
||||
|
||||
|
||||
define('NORMAL', 1);
|
||||
define('DEBUG', 5);
|
||||
|
||||
$config = [];
|
||||
|
||||
|
||||
/*
|
||||
* you can override any of these values by putting the
|
||||
* variable to be overridden in config-site.php
|
||||
*/
|
||||
|
||||
$config['ENABLE_MOBILE_PREVIEW'] = 0;
|
||||
|
||||
$config['SITE_LOGO_LG'] = '/view/theme/default/assets/images/archive-logo-lg.png';
|
||||
$config['SITE_LOGO_SM'] = '/view/theme/default/assets/images/archive-logo-sm.png';
|
||||
$config['COMPATIBILITY'] = 'Which browsers are supported, etc';
|
||||
|
||||
$config['BRANDING_TEXT'] = '';
|
||||
$config['BRANDING_URL'] = '';
|
||||
$config['BRANDING_LOGO'] = $config['SITE_LOGO_SM'];
|
||||
$config['BRANDING_BACKGROUND_COLOUR'] = '';
|
||||
$config['BRANDING_TEXT_COLOUR'] = '';
|
||||
$config['BRANDING_FAVICON'] = '/view/theme/default/assets/ico/favicon.png';
|
||||
$config['SUPPORT_LINK'] = '';
|
||||
$config['TITLE_PREFIX'] = '';
|
||||
|
||||
$config['CUSTOM_PRE_AUTH_FUNCTION'] = '';
|
||||
$config['CUSTOM_EMAIL_QUERY_FUNCTION'] = '';
|
||||
|
||||
$config['DOMAIN_REGEX'] = '/^[a-zA-Z0-9]+[a-zA-Z0-9-_\.]{0,}\.[a-zA-Z0-9]{2,20}$/';
|
||||
|
||||
$config['BOOTSTRAP_THEME'] = '-cosmo';
|
||||
|
||||
$config['DEFAULT_LANG'] = 'en';
|
||||
|
||||
$config['THEME'] = 'default';
|
||||
|
||||
$config['SITE_NAME'] = 'piler.example.com';
|
||||
$config['SITE_URL'] = 'http://piler.example.com/';
|
||||
|
||||
$config['EXTERNAL_DASHBOARD_URL'] = '';
|
||||
|
||||
$config['SESSION_EXPIRY'] = 3600;
|
||||
$config['DELTA_INDEXER_PERIOD'] = 1800;
|
||||
|
||||
$config['ENABLE_SAAS'] = 0;
|
||||
$config['CAPTCHA_FAILED_LOGIN_COUNT'] = 0;
|
||||
|
||||
$config['ENABLE_REFERENCES'] = 1;
|
||||
$config['ENABLE_DOWNLOADING_ALL_SEARCH_HITS'] = 0;
|
||||
$config['EML_NAME_BASED_ON_SUBJECT'] = 1;
|
||||
|
||||
$config['ENABLE_TABLE_RESIZE'] = 0;
|
||||
$config['DEMO_MODE'] = 0;
|
||||
|
||||
$config['SHOW_MENU_FOR_OUTLOOK'] = 0;
|
||||
|
||||
$config['SEARCH_QUERY_QUOTING'] = 0;
|
||||
|
||||
$config['LOCALIZE_MESSAGE_HEADERS_IN_PREVIEW'] = 1;
|
||||
|
||||
$config['TIMEZONE'] = 'Europe/Budapest';
|
||||
|
||||
$config['PROVIDED_BY'] = 'www.mailpiler.org';
|
||||
$config['SITE_KEYWORDS'] = 'piler email archiver';
|
||||
$config['SITE_DESCRIPTION'] = 'piler email archiver';
|
||||
|
||||
$config['INDEXER_BEACON'] = '/var/piler/stat/indexer';
|
||||
$config['PURGE_BEACON'] = '/var/piler/stat/purge';
|
||||
|
||||
$config['ENABLE_PDF_DOWNLOAD'] = 0;
|
||||
// You may need to run "Xvfb :1 -screen 0 1024x768x16"
|
||||
// In this case specify WKHTMLTOPDF_COMMAND = "DISPLAY=:1.0 wkhtmltopdf";
|
||||
$config['WKHTMLTOPDF_COMMAND'] = "wkhtmltopdf";
|
||||
|
||||
// authentication against an ldap directory (disabled by default)
|
||||
|
||||
$config['ENABLE_LDAP_AUTH'] = 0;
|
||||
$config['LDAP_HOST'] = 'ldap.example.com';
|
||||
$config['LDAP_HELPER_DN'] = 'cn=....';
|
||||
$config['LDAP_HELPER_PASSWORD'] = 'xxxxxxx';
|
||||
$config['LDAP_MAIL_ATTR'] = 'mail';
|
||||
$config['LDAP_AUDITOR_MEMBER_DN'] = '';
|
||||
$config['LDAP_ADMIN_MEMBER_DN'] = '';
|
||||
$config['LDAP_BASE_DN'] = '';
|
||||
$config['LDAP_USE_START_TLS'] = 0;
|
||||
|
||||
// AD specific settings
|
||||
//
|
||||
$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'user';
|
||||
$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'group';
|
||||
$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'member';
|
||||
$config['LDAP_MAIL_ATTR'] = 'proxyAddresses';
|
||||
|
||||
// zimbra specific settings
|
||||
//$config['LDAP_HELPER_DN'] = 'uid=zimbra,cn=admins,cn=zimbra';
|
||||
//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'zimbraAccount';
|
||||
//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'zimbraDistributionList';
|
||||
//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'zimbraMailForwardingAddress';
|
||||
//$config['LDAP_MAIL_ATTR'] = 'mail';
|
||||
|
||||
// Lotus Notes specific settings for ldap authentication
|
||||
//
|
||||
//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'dominoPerson';
|
||||
//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'dominoGroup');
|
||||
//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mail';
|
||||
//$config['LDAP_MAIL_ATTR'] = 'mail';
|
||||
|
||||
// iredmail specific settings
|
||||
//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=example,dc=com';
|
||||
//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'mailUser';
|
||||
//$config['LDAP_BASE_DN'] = 'o=domains,dc=example,dc=com';
|
||||
//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'mailList';
|
||||
//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'memberOfGroup';
|
||||
//$config['LDAP_MAIL_ATTR'] = 'mail';
|
||||
|
||||
// Uninvention specific settings
|
||||
//$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
|
||||
//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
|
||||
//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
|
||||
//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
|
||||
|
||||
|
||||
// enable single sign-on (disabled by default)
|
||||
$config['ENABLE_SSO_LOGIN'] = 0;
|
||||
|
||||
$config['STRIP_DOMAIN_NAME_FROM_USERNAME'] = 0;
|
||||
|
||||
// enable authentication against an imap server (disabled by default)
|
||||
|
||||
$config['ENABLE_IMAP_AUTH'] = 0;
|
||||
$config['RESTORE_OVER_IMAP'] = 0;
|
||||
$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
|
||||
$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
|
||||
$config['IMAP_HOST'] = 'mail.example.com';
|
||||
$config['IMAP_PORT'] = 993;
|
||||
$config['IMAP_SSL'] = true;
|
||||
|
||||
// enable authentication against a pop3 server (disabled by default)
|
||||
|
||||
$config['ENABLE_POP3_AUTH'] = 0;
|
||||
$config['POP3_HOST'] = 'mail.example.com';
|
||||
$config['POP3_PORT'] = 995;
|
||||
$config['POP3_SSL'] = true;
|
||||
|
||||
$config['ENABLE_GOOGLE_AUTHENTICATOR'] = 1;
|
||||
|
||||
// enable authentication against google (disabled by default)
|
||||
// see http://www.mailpiler.org/en/google-apps-free.html for details
|
||||
|
||||
$config['ENABLE_GOOGLE_LOGIN'] = 0;
|
||||
$config['GOOGLE_CLIENT_ID'] = 'xxxxxxxxxxx';
|
||||
$config['GOOGLE_CLIENT_SECRET'] = 'xxxxxxxxxxxxx';
|
||||
$config['GOOGLE_DEVELOPER_KEY'] = 'xxxxxxxxxxxx';
|
||||
$config['GOOGLE_APPLICATION_NAME'] = 'piler enterprise email archiver';
|
||||
$config['GOOGLE_ALL_MAIL'] = '[Gmail]/All Mail';
|
||||
|
||||
|
||||
$config['ENABLE_AUDIT'] = 1;
|
||||
$config['MEMCACHED_ENABLED'] = 0;
|
||||
$config['PASSWORD_CHANGE_ENABLED'] = 1;
|
||||
$config['ENABLE_STATISTICS'] = 1;
|
||||
$config['ENABLE_HISTORY'] = 1;
|
||||
$config['ENABLE_DELETE'] = 0;
|
||||
$config['NEED_TO_APPROVE_DELETE'] = 0;
|
||||
$config['ENABLE_REMOTE_IMAGES'] = '0';
|
||||
$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 0;
|
||||
$config['ENABLE_LDAP_IMPORT_FEATURE'] = 0;
|
||||
$config['ENABLE_FOLDER_RESTRICTIONS'] = 0;
|
||||
|
||||
// When enabled the search page auto loads the search results
|
||||
// ie. as if the user had clicked on the Search button
|
||||
$config['ENABLE_INSTANT_SEARCH'] = 0;
|
||||
$config['SEARCH_RESULT_CHECKBOX_CHECKED'] = 0;
|
||||
$config['HELPER_URL'] = '';
|
||||
$config['LOG_LEVEL'] = NORMAL;
|
||||
$config['REWRITE_MESSAGE_ID'] = 0;
|
||||
$config['RESTORE_EMAILS_AS_ATTACHMENT'] = 0;
|
||||
$config['RESTRICTED_AUDITOR'] = 0;
|
||||
|
||||
$config['SHOW_ENVELOPE_JOURNAL'] = 0;
|
||||
$config['BULK_DOWNLOAD_FOR_USERS'] = 1;
|
||||
$config['MAX_DOWNLOAD_PER_HOUR'] = 0;
|
||||
$config['MAX_RESTORE_PER_HOUR'] = 0;
|
||||
|
||||
$config['MAX_CGI_FROM_SUBJ_LEN'] = 48;
|
||||
$config['PAGE_LEN'] = 20;
|
||||
$config['MAX_NUMBER_OF_FROM_ITEMS'] = 5;
|
||||
$config['MAX_SEARCH_HITS'] = 1000;
|
||||
|
||||
$config['SPHINX_MAIN_INDEX_THRESHOLD'] = 2000000000;
|
||||
|
||||
$config['DEFAULT_RETENTION'] = 0;
|
||||
|
||||
$config['LOCALHOST'] = '127.0.0.1';
|
||||
$config['PILER_HOST'] = '0.0.0.0';
|
||||
$config['PILER_PORT'] = 25;
|
||||
$config['SMARTHOST'] = '';
|
||||
$config['SMARTHOST_PORT'] = 25;
|
||||
$config['SMARTHOST_USER'] = '';
|
||||
$config['SMARTHOST_PASSWORD'] = '';
|
||||
$config['SMTP_DOMAIN'] = 'example.com';
|
||||
$config['SMTP_FROMADDR'] = 'no-reply@example.com';
|
||||
$config['ADMIN_EMAIL'] = 'admin@example.com';
|
||||
|
||||
$config['PILER_HEADER_FIELD'] = 'X-piler-id: ';
|
||||
|
||||
$config['DEFAULT_POLICY'] = 'default_policy';
|
||||
|
||||
$config['DIR_BASE'] = dirname(__FILE__) . '/';
|
||||
|
||||
$config['DIR_SPHINX'] = '/var/piler/manticore/';
|
||||
$config['DIR_STAT'] = '/var/piler/stat';
|
||||
$config['DIR_IMAP'] = '/var/piler/imap';
|
||||
$config['DIR_TMP'] = '/var/piler/tmp';
|
||||
|
||||
$config['DECRYPT_BINARY'] = 'BINDIR/pilerget';
|
||||
$config['DECRYPT_ATTACHMENT_BINARY'] = 'BINDIR/pileraget';
|
||||
$config['DECRYPT_BUFFER_LENGTH'] = 65536;
|
||||
|
||||
$config['OPENSSL_BINARY'] = '/usr/bin/openssl';
|
||||
$config['TSA_URL'] = '';
|
||||
$config['TSA_PUBLIC_KEY_FILE'] = '';
|
||||
$config['TSA_START_ID'] = 1;
|
||||
$config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
|
||||
$config['TSA_VERIFY_CERTIFICATE'] = true;
|
||||
$config['TSA_RELAXED_CHECK'] = false;
|
||||
$config['TSA_AUTH_USER'] = '';
|
||||
$config['TSA_AUTH_PASSWORD'] = '';
|
||||
$config['TSA_AUTH_CERT_FILE'] = '';
|
||||
$config['TSA_AUTH_KEY_FILE'] = '';
|
||||
$config['TSA_AUTH_KEY_PASSWORD'] = '';
|
||||
|
||||
$config['DB_DRIVER'] = 'mysql';
|
||||
$config['DB_PREFIX'] = '';
|
||||
$config['DB_HOSTNAME'] = 'localhost';
|
||||
$config['DB_USERNAME'] = 'piler';
|
||||
$config['DB_PASSWORD'] = 'piler';
|
||||
$config['DB_DATABASE'] = 'piler';
|
||||
$config['DB_CHARSET'] = 'utf8mb4';
|
||||
|
||||
$config['SPHINX_DRIVER'] = 'sphinx';
|
||||
$config['SPHINX_DATABASE'] = 'sphinx';
|
||||
$config['SPHINX_HOSTNAME'] = '127.0.0.1:9306';
|
||||
$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9307';
|
||||
$config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
|
||||
$config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
|
||||
$config['SPHINX_TAG_INDEX'] = 'tag1';
|
||||
$config['SPHINX_NOTE_INDEX'] = 'note1';
|
||||
$config['SPHINX_STRICT_SCHEMA'] = 1;
|
||||
$config['RT'] = 0;
|
||||
$config['MAX_EMAIL_LEN'] = 41;
|
||||
|
||||
$config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload';
|
||||
$config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r';
|
||||
$config['CPU_USAGE_COMMAND'] = "LC_ALL=C mpstat | tail -1 | rev | awk '{ print $1 }' | rev";
|
||||
$config['PILER_BINARY'] = "SBINDIR/piler";
|
||||
|
||||
$config['LDAP_IMPORT_CONFIG_FILE'] = '/usr/local/etc/ldap-import.cfg';
|
||||
|
||||
$config['DN_MAX_LEN'] = 255;
|
||||
$config['USE_EMAIL_AS_USERNAME'] = 1;
|
||||
$config['LDAP_IMPORT_MINIMUM_NUMBER_OF_USERS_TO_HEALTH_OK'] = 100;
|
||||
|
||||
$config['HEADER_LINE_TO_HIDE'] = 'X-Envelope-To:';
|
||||
|
||||
$config['PILER_LOGIN_HELPER_PLACEHOLDER'] = 'PILER_COMMENT_FOR_PROPER_LOGIN_SO_THIS_CAN_BE_ANYTHING_JUST_BE_IT_SOMETHING_LIKE_A_UNIQUE_VALUE';
|
||||
|
||||
|
||||
$config['SIZE_X'] = 430;
|
||||
$config['SIZE_Y'] = 250;
|
||||
|
||||
$config['DATE_TEMPLATE'] = 'Y.m.d';
|
||||
$config['DATE_FORMAT'] = 'YYYY-MM-DD';
|
||||
$config['JQUERY_DATE_FORMAT'] = 'yy-mm-dd';
|
||||
$config['DECIMAL_SEPARATOR'] = "."; // See https://www.php.net/manual/en/function.number-format
|
||||
$config['THOUSANDS_SEPARATOR'] = ","; // for the format options
|
||||
|
||||
$config['ENABLE_GB2312_FIX'] = 1;
|
||||
|
||||
$config['FROM_LENGTH_TO_SHOW'] = 28;
|
||||
|
||||
$config['HISTORY_REFRESH'] = 60;
|
||||
$config['HEALTH_REFRESH'] = 300;
|
||||
$config['HEALTH_RATIO'] = 80;
|
||||
|
||||
$config['MAX_AUDIT_HITS'] = 1000;
|
||||
|
||||
$config['MIN_PASSWORD_LENGTH'] = 6;
|
||||
|
||||
$config['MIN_PREFIX_LEN'] = 5;
|
||||
|
||||
$config['ADMIN_CAN_POWER_SEARCH'] = 0;
|
||||
$config['FOUR_EYES_LOGIN_FOR_AUDITOR'] = 0;
|
||||
|
||||
$config['MEMCACHED_PREFIX'] = '_piler:';
|
||||
$config['MEMCACHED_TTL'] = 900;
|
||||
|
||||
$config['JS_CODE'] = '
|
||||
<script src="//code.jquery.com/jquery-2.2.4.min.js" integrity="sha256-BbhdlvQf/xTY9gja0Dq3HiwQF8LaCRTXxZKRutelT44=" crossorigin="anonymous"></script>
|
||||
<script src="//code.jquery.com/ui/1.12.1/jquery-ui.min.js" integrity="sha256-VazP97ZCwtekAsvgPBSUwPFKdrwD3unUfSGVYrahUqU=" crossorigin="anonymous"></script>
|
||||
<script src="//stackpath.bootstrapcdn.com/bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha384-aJ21OjlMXNL5UyIl/XNwTMqvzeRMZH2w8c5cRVpzpU8Y5bApTppSuUkhZXN0VxHd" crossorigin="anonymous"></script>
|
||||
|
||||
<script type="text/javascript" src="/view/javascript/rc-splitter.js"></script>
|
||||
<script type="text/javascript" src="/view/javascript/piler.js"></script>
|
||||
';
|
||||
|
||||
$config['CSS_CODE'] = '
|
||||
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
|
||||
<link rel="stylesheet" href="/view/theme/default/assets/css/piler.css" />
|
||||
';
|
||||
|
||||
$SUPPRESS_RECIPIENTS = array();
|
||||
|
||||
|
||||
$memcached_servers = array(
|
||||
array('127.0.0.1', 11211)
|
||||
);
|
||||
|
||||
$partitions_to_monitor = array('/', '/home', '/var', '/var/piler', '/tmp');
|
||||
$config['DATA_PARTITION'] = '/var';
|
||||
|
||||
$config['DELIMITER'] = "\t";
|
||||
|
||||
$config['TRACKING_CODE'] = '';
|
||||
|
||||
$mailattrs = ["mail", "mailalternateaddress", "proxyaddresses", "zimbraMailForwardingAddress", "member", "memberOfGroup", "othermailbox", "mailprimaryaddress", "mailalternativeaddress"];
|
||||
|
||||
$langs = array(
|
||||
'cz',
|
||||
'de',
|
||||
'en',
|
||||
'es',
|
||||
'fr',
|
||||
'hu',
|
||||
'it',
|
||||
'pl',
|
||||
'pt',
|
||||
'ru',
|
||||
'tr',
|
||||
'tw'
|
||||
);
|
||||
|
||||
|
||||
$themes = array(
|
||||
'default',
|
||||
);
|
||||
|
||||
|
||||
$paging = array(
|
||||
10,
|
||||
20,
|
||||
30,
|
||||
50
|
||||
);
|
||||
|
||||
$automated_search_recipients = array();
|
||||
|
||||
|
||||
$letters = array('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z');
|
||||
|
||||
|
||||
define('NOW', time());
|
||||
|
||||
|
||||
/*
|
||||
* normally you don't have to change anything below
|
||||
*/
|
||||
|
||||
if(file_exists('SYSCONFDIR/piler/config-site.php')) { require_once 'SYSCONFDIR/piler/config-site.php'; }
|
||||
|
||||
ini_set('session.cookie_lifetime', $config['SESSION_EXPIRY']);
|
||||
|
||||
require($config['DIR_BASE'] . "/system/registry.php");
|
||||
require($config['DIR_BASE'] . "/system/request.php");
|
||||
|
||||
$session = new Session();
|
||||
Registry::set("session", $session);
|
||||
|
||||
|
||||
if($session->get("theme") && preg_match("/^([a-zA-Z0-9\-\_]+)$/", $session->get("theme"))) { $config['THEME'] = $session->get("theme"); }
|
||||
|
||||
include("system/helper/detectmobilebrowser.php");
|
||||
|
||||
// make sure auditors are restricted in a saas environment
|
||||
if($config['ENABLE_SAAS'] == 1) { $config['RESTRICTED_AUDITOR'] = 1; }
|
||||
if($session->get("username") == 'auditor@local' || isset($_SERVER['argv'][2]) ) { $config['RESTRICTED_AUDITOR'] = 0; }
|
||||
|
||||
|
||||
|
||||
foreach ($config as $k => $v) {
|
||||
define($k, $v);
|
||||
}
|
||||
|
||||
|
||||
define('TABLE_USER', 'user');
|
||||
define('TABLE_GROUP', 'usergroup');
|
||||
define('TABLE_GROUP_USER', 'group_user');
|
||||
define('TABLE_GROUP_EMAIL', 'group_email');
|
||||
define('TABLE_FOLDER', 'folder');
|
||||
define('TABLE_FOLDER_USER', 'folder_user');
|
||||
define('TABLE_FOLDER_EXTRA', 'folder_extra');
|
||||
define('TABLE_FOLDER_MESSAGE', 'folder_message');
|
||||
define('TABLE_EMAIL', 'email');
|
||||
define('TABLE_META', 'metadata');
|
||||
define('TABLE_RCPT', 'rcpt');
|
||||
define('TABLE_ATTACHMENT', 'attachment');
|
||||
define('TABLE_SEARCH', 'search');
|
||||
define('TABLE_EMAIL_LIST', 'email_groups');
|
||||
define('TABLE_TAG', 'tag');
|
||||
define('TABLE_NOTE', '`note`');
|
||||
define('TABLE_USER_SETTINGS', 'user_settings');
|
||||
define('TABLE_REMOTE', 'remote');
|
||||
define('TABLE_DOMAIN', 'domain');
|
||||
define('TABLE_DOMAIN_USER', 'domain_user');
|
||||
define('TABLE_COUNTER', 'counter');
|
||||
define('TABLE_COUNTER_MSG', 'counter_messages');
|
||||
define('TABLE_STAT_COUNTER', 'counter_stats');
|
||||
define('TABLE_AUDIT', 'audit');
|
||||
define('TABLE_ARCHIVING_RULE', 'archiving_rule');
|
||||
define('TABLE_FOLDER_RULE', 'folder_rule');
|
||||
define('TABLE_RETENTION_RULE', 'retention_rule');
|
||||
define('TABLE_OPTION', 'option');
|
||||
define('TABLE_LDAP', 'ldap');
|
||||
define('TABLE_CUSTOMER_SETTINGS', 'customer_settings');
|
||||
define('TABLE_ONLINE', 'online');
|
||||
define('TABLE_IMPORT', 'import');
|
||||
define('TABLE_GOOGLE', 'google');
|
||||
define('TABLE_GOOGLE_IMAP', 'google_imap');
|
||||
define('TABLE_AUTOSEARCH', 'autosearch');
|
||||
define('TABLE_LEGAL_HOLD', 'legal_hold');
|
||||
define('TABLE_TIMESTAMP', 'timestamp');
|
||||
define('TABLE_PRIVATE', 'private');
|
||||
define('TABLE_DELETED', 'deleted');
|
||||
define('VIEW_MESSAGES', 'v_messages');
|
||||
|
||||
define('EOL', "\r\n");
|
||||
|
||||
define('DIR_SYSTEM', DIR_BASE . 'system/');
|
||||
define('DIR_MODEL', DIR_BASE . 'model/');
|
||||
define('DIR_DATABASE', DIR_BASE . 'system/database/');
|
||||
define('DIR_IMAGE', DIR_BASE . 'image/');
|
||||
define('DIR_LANGUAGE', DIR_BASE . 'language/');
|
||||
define('DIR_APPLICATION', DIR_BASE . 'controller/');
|
||||
define('DIR_THEME', DIR_BASE . 'view/theme/');
|
||||
define('DIR_REPORT', DIR_BASE . 'reports/');
|
||||
define('DIR_LOG', DIR_BASE . 'log/');
|
||||
|
||||
define('REMOTE_IMAGE_REPLACEMENT', '/view/theme/' . THEME . '/images/remote.gif');
|
||||
define('ICON_ARROW_UP', '/view/theme/' . THEME . '/images/arrowup.gif');
|
||||
define('ICON_ARROW_DOWN', '/view/theme/' . THEME . '/images/arrowdown.gif');
|
||||
define('ICON_ATTACHMENT', '/view/theme/' . THEME . '/images/attachment_icon.png');
|
||||
define('ICON_TAG', '/view/theme/' . THEME . '/images/tag_blue.png');
|
||||
define('ICON_GREEN_OK', '/view/theme/' . THEME . '/images/green_ok.png');
|
||||
define('ICON_RED_X', '/view/theme/' . THEME . '/images/red_x.png');
|
||||
define('ICON_DOWNLOAD', '/view/theme/' . THEME . '/images/download_icon.jpg');
|
||||
define('ICON_NOTES', '/view/theme/' . THEME . '/images/notes.png');
|
||||
define('ICON_PLUS', '/view/theme/' . THEME . '/images/plus.gif');
|
||||
define('ICON_MINUS', '/view/theme/' . THEME . '/images/minus.gif');
|
||||
define('ICON_EMPTY', '/view/theme/' . THEME . '/images/1x1.gif');
|
||||
|
||||
define('QSHAPE_ACTIVE_INCOMING', DIR_STAT . '/active+incoming');
|
||||
define('QSHAPE_ACTIVE_INCOMING_SENDER', DIR_STAT . '/active+incoming-sender');
|
||||
define('QSHAPE_DEFERRED', DIR_STAT . '/deferred');
|
||||
define('QSHAPE_DEFERRED_SENDER', DIR_STAT . '/deferred-sender');
|
||||
|
||||
define('CPUSTAT', DIR_STAT . '/cpu.stat');
|
||||
define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat');
|
||||
define('ARCHIVE_SIZE', DIR_STAT . '/archive.size');
|
||||
define('SPHINX_CURRENT_MAIN_INDEX_SIZE', DIR_STAT . '/current_main_index_size');
|
||||
define('SPHINX_TOTAL_INDEX_SIZE', DIR_STAT . '/total_index_size');
|
||||
define('LOCK_FILE', DIR_LOG . 'lock');
|
||||
|
||||
define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php');
|
||||
define('AUDIT_HELPER_URL', SITE_URL . 'audit-helper.php');
|
||||
define('BULK_RESTORE_URL', SITE_URL . 'bulkrestore.php');
|
||||
define('SAVE_SEARCH_URL', SITE_URL . 'index.php?route=search/save');
|
||||
define('LOAD_SAVED_SEARCH_URL', SITE_URL . 'index.php?route=search/load');
|
||||
define('SEARCH_TAG_URL', SITE_URL . 'index.php?route=search/tag');
|
||||
define('MESSAGE_NOTE_URL', SITE_URL . 'index.php?route=message/note');
|
||||
define('GOOGLE_REDIRECT_URL', SITE_URL . 'google.php');
|
||||
define('HEALTH_URL', SITE_URL . 'index.php?route=health/health');
|
||||
define('HEALTH_WORKER_URL', SITE_URL . 'index.php?route=health/worker');
|
||||
|
||||
define('LDAP_TYPE_GENERIC', 'generic_ldap');
|
||||
|
||||
define('ATTACHMENT_DUMP_CHECKPOINT', 'attachment_dump_checkpoint');
|
||||
|
||||
define('ACTION_ALL', 0);
|
||||
define('ACTION_UNKNOWN', 1);
|
||||
define('ACTION_LOGIN', 2);
|
||||
define('ACTION_LOGIN_FAILED', 3);
|
||||
define('ACTION_LOGOUT', 4);
|
||||
define('ACTION_VIEW_MESSAGE', 5);
|
||||
define('ACTION_VIEW_HEADER', 6);
|
||||
define('ACTION_UNAUTHORIZED_VIEW_MESSAGE', 7);
|
||||
define('ACTION_RESTORE_MESSAGE', 8);
|
||||
define('ACTION_DOWNLOAD_MESSAGE', 9);
|
||||
define('ACTION_SEARCH', 10);
|
||||
define('ACTION_SAVE_SEARCH', 11);
|
||||
define('ACTION_CHANGE_USER_SETTINGS', 12);
|
||||
define('ACTION_REMOVE_MESSAGE', 13);
|
||||
define('ACTION_UNAUTHORIZED_REMOVE_MESSAGE', 14);
|
||||
define('ACTION_DOWNLOAD_ATTACHMENT', 15);
|
||||
define('ACTION_UNAUTHORIZED_DOWNLOAD_ATTACHMENT', 16);
|
||||
define('ACTION_VIEW_JOURNAL', 17);
|
||||
define('ACTION_NOT_SPAM', 18);
|
||||
define('ACTION_MARK_AS_PRIVATE', 19);
|
||||
define('ACTION_MARK_MESSAGE_FOR_REMOVAL', 20);
|
||||
define('ACTION_REJECT_REMOVAL', 21);
|
||||
|
||||
$actions = array(
|
||||
'unknown' => 1,
|
||||
'login' => 2,
|
||||
'loginfailed' => 3,
|
||||
'logout' => 4,
|
||||
'view' => 5,
|
||||
'view_header' => 6,
|
||||
'restore' => 8,
|
||||
'download' => 9,
|
||||
'search' => 10,
|
||||
'save_search' => 11,
|
||||
'download_attachment' => 15,
|
||||
'journal' => 17,
|
||||
'private' => 19,
|
||||
'marked_for_removal', 20,
|
||||
'reject_removal', 21
|
||||
);
|
||||
|
||||
|
||||
$import_status = array(
|
||||
0 => 'PENDING',
|
||||
1 => 'RUNNING',
|
||||
2 => 'FINISHED',
|
||||
3 => 'ERROR'
|
||||
);
|
||||
|
||||
|
||||
$counters = array(MEMCACHED_PREFIX . 'rcvd', MEMCACHED_PREFIX . 'virus', MEMCACHED_PREFIX . 'duplicate', MEMCACHED_PREFIX . 'ignore', MEMCACHED_PREFIX . 'counters_last_update');
|
||||
|
||||
if(!isset($health_smtp_servers)) {
|
||||
$health_smtp_servers = array( array(PILER_HOST, PILER_PORT, "piler"), array(SMARTHOST, SMARTHOST_PORT, "smarthost") );
|
||||
}
|
||||
|
||||
if(SPHINX_STRICT_SCHEMA) {
|
||||
define('FROM_TOKEN', '@sender');
|
||||
define('FROMDOMAIN_TOKEN', '@senderdomain');
|
||||
define('TO_TOKEN', '@rcpt');
|
||||
define('TODOMAIN_TOKEN', '@rcptdomain');
|
||||
} else {
|
||||
define('FROM_TOKEN', '@from');
|
||||
define('FROMDOMAIN_TOKEN', '@fromdomain');
|
||||
define('TO_TOKEN', '@to');
|
||||
define('TODOMAIN_TOKEN', '@todomain');
|
||||
}
|
||||
256
configure.in
256
configure.in
@ -24,9 +24,9 @@ AC_SUBST(OBJS)
|
||||
AC_SUBST(RUNNING_USER)
|
||||
AC_SUBST(SUBDIRS)
|
||||
AC_SUBST(MAKE)
|
||||
AC_SUBST(sql_includes)
|
||||
AC_SUBST(sql_libs)
|
||||
AC_SUBST(sql_obj)
|
||||
AC_SUBST(mysql_includes)
|
||||
AC_SUBST(mysql_libs)
|
||||
AC_SUBST(mysql_obj)
|
||||
AC_SUBST(libclamav_extra_libs)
|
||||
AC_SUBST(id_bin)
|
||||
AC_SUBST(iv)
|
||||
@ -37,10 +37,10 @@ have_clamd="no"
|
||||
|
||||
have_antivirus="no"
|
||||
have_mysql="no"
|
||||
have_psql="no"
|
||||
have_tre="no"
|
||||
have_zip="no"
|
||||
have_zlib="no"
|
||||
have_starttls="no"
|
||||
have_tcpwrappers="no"
|
||||
have_tweak_sent_time="no"
|
||||
|
||||
@ -50,18 +50,15 @@ catppt="no"
|
||||
ppthtml="no"
|
||||
xls2csv="no"
|
||||
unrtf="no"
|
||||
tnef="no"
|
||||
|
||||
timeout_binary=""
|
||||
|
||||
have_static_build="no"
|
||||
have_compat_storage_layout="no"
|
||||
|
||||
antispam_libs="-lz -lm -ldl -lcrypto -lssl"
|
||||
defs=""
|
||||
objs=""
|
||||
user_obj=""
|
||||
sql_obj=""
|
||||
mysql_obj=""
|
||||
os=`uname -s`
|
||||
|
||||
id_bin="id"
|
||||
@ -80,12 +77,12 @@ if test -f `which gcc`; then gcc -v 2> aa; GCC_VERSION=`tail -1 aa`; rm -f aa; f
|
||||
if test "$os" = "SunOS"; then
|
||||
echo "#define CONFIGURE_PARAMS \"Build Date: "`date`'\\n" \\' > $CONFIGURE_PARAMS_FILE
|
||||
else
|
||||
echo "#define CONFIGURE_PARAMS \"Build Date: "`date`"\\nldd version: $LDD_VERSION\\ngcc version: $GCC_VERSION\\nOS: "`uname -a`"\n"\" \\ > $CONFIGURE_PARAMS_FILE
|
||||
echo "#define CONFIGURE_PARAMS \"Build Date: "`date`"\\nldd version: $LDD_VERSION\\ngcc version: $GCC_VERSION\n"\" \\ > $CONFIGURE_PARAMS_FILE
|
||||
fi
|
||||
echo "\"Configure command: ./configure $PARAMS\"" >> $CONFIGURE_PARAMS_FILE
|
||||
|
||||
|
||||
SUBDIRS="src etc util init.d systemd unit_tests webui"
|
||||
SUBDIRS="src etc util init.d test"
|
||||
|
||||
|
||||
dnl static build
|
||||
@ -94,12 +91,6 @@ AC_ARG_ENABLE(static-build,
|
||||
[ --enable-static-build build statically linked executables (default: dynamically linked)], have_static_build=$enableval, have_static_build="no")
|
||||
|
||||
|
||||
dnl
|
||||
|
||||
AC_ARG_ENABLE(compat-layout,
|
||||
[ --enable-compat-layout support for older storage layout (default: no)], have_compat_storage_layout=$enableval, have_compat_storage_layout="no")
|
||||
|
||||
|
||||
dnl clamd
|
||||
|
||||
|
||||
@ -115,13 +106,22 @@ AC_ARG_ENABLE(memcached,
|
||||
[ --enable-memcached build memcached support], want_memcached=$enableval, want_memcached="no")
|
||||
|
||||
|
||||
dnl tcpwrappers library
|
||||
AC_ARG_ENABLE(starttls,
|
||||
[ --enable-starttls build starttls support], have_starttls=$enableval, have_starttls="no")
|
||||
|
||||
AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, echo "tcpd.h is not found")
|
||||
AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
|
||||
|
||||
if test "$have_tcpwrappers" = "no"; then
|
||||
echo "can't find either tcpd.h or libwrap. Tcpwrappers support is disabled";
|
||||
AC_ARG_ENABLE(tcpwrappers,
|
||||
[ --enable-tcpwrappers build tcpwrappers support], want_tcpwrappers=$enableval, want_tcpwrappers="no")
|
||||
|
||||
|
||||
if test "$want_tcpwrappers" = "yes"; then
|
||||
AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, have_tcpwrappers=no)
|
||||
AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
|
||||
|
||||
if test "$have_tcpwrappers" = "no"; then
|
||||
echo "can't find either tcpd.h or libwrap";
|
||||
exit 1;
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@ -176,90 +176,24 @@ if test "$have_zlib" = "no"; then
|
||||
fi
|
||||
|
||||
|
||||
AC_ARG_WITH(database,
|
||||
[ --with-database[[=mysql]] select the used database, currently mysql only (default: none)],[
|
||||
if test "$withval" != "no" -a "$withval" != "yes"; then
|
||||
DATABASE=$withval
|
||||
fi
|
||||
AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, yes)
|
||||
|
||||
|
||||
if test "$withval" = "mysql"; then
|
||||
|
||||
AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, yes)
|
||||
|
||||
if test x$MYSQL_CONFIG = xyes; then
|
||||
have_mysql="yes"
|
||||
fi
|
||||
|
||||
if test "$have_mysql" = "yes"; then
|
||||
|
||||
sql_includes=`mysql_config --cflags`
|
||||
sql_libs=`mysql_config --libs_r`
|
||||
sql_obj="mysql.o"
|
||||
|
||||
AC_CHECK_LIB([guide],[main],[AC_CHECK_LIB(guide, _intel_fast_memset, have_icc_guide=yes, have_icc_guide=no)],[],[])ac_cv_lib_guide=ac_cv_lib_guide_main
|
||||
|
||||
else
|
||||
echo "MySQL support is not found"
|
||||
exit 1;
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
if test "$withval" = "mariadb"; then
|
||||
|
||||
AC_CHECK_PROG(MYSQL_CONFIG, mariadb_config, yes)
|
||||
|
||||
if test x$MYSQL_CONFIG = xyes; then
|
||||
have_mysql="yes"
|
||||
fi
|
||||
|
||||
if test "$have_mysql" = "yes"; then
|
||||
|
||||
sql_includes=`mariadb_config --cflags`
|
||||
sql_libs=`mariadb_config --libs_r`
|
||||
sql_obj="mysql.o"
|
||||
|
||||
else
|
||||
echo "mariadb_config is not found, please install \"MariaDB Client Library for C 1.0.0 Stable\", see https://downloads.mariadb.org/client-native/1.0.0/"
|
||||
exit 1;
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
if test "$withval" = "psql"; then
|
||||
|
||||
AC_CHECK_PROG(PSQL_CONFIG, pg_config, yes)
|
||||
|
||||
if test x$PSQL_CONFIG = xyes; then
|
||||
have_psql="yes"
|
||||
fi
|
||||
|
||||
if test "$have_psql" = "yes"; then
|
||||
|
||||
sql_includes="-I`pg_config --includedir`"
|
||||
sql_libs="-L`pg_config --libdir`"
|
||||
sql_obj="psql.o"
|
||||
|
||||
else
|
||||
echo "PSQL support is not found"
|
||||
exit 1;
|
||||
fi
|
||||
fi
|
||||
|
||||
])
|
||||
|
||||
|
||||
dnl timeout binary
|
||||
|
||||
if test z`which timeout 2>/dev/null` != "z"; then
|
||||
timeout_binary=`which timeout`
|
||||
AC_DEFINE_UNQUOTED(TIMEOUT_BINARY, "$timeout_binary", [timeout binary])
|
||||
if test x$MYSQL_CONFIG = xyes; then
|
||||
have_mysql="yes"
|
||||
fi
|
||||
|
||||
if test "$have_mysql" = "yes"; then
|
||||
|
||||
mysql_includes=`mysql_config --cflags`
|
||||
mysql_libs=`mysql_config --libs_r`
|
||||
|
||||
AC_CHECK_LIB([guide],[main],[AC_CHECK_LIB(guide, _intel_fast_memset, have_icc_guide=yes, have_icc_guide=no)],[],[])ac_cv_lib_guide=ac_cv_lib_guide_main
|
||||
|
||||
else
|
||||
echo "MySQL support is not found"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
read -r version < VERSION
|
||||
AC_DEFINE_UNQUOTED(VERSION, "$version")
|
||||
|
||||
|
||||
dnl user running piler
|
||||
@ -313,63 +247,16 @@ DATADIR=$data_dir
|
||||
AC_SUBST(DATADIR)
|
||||
AC_DEFINE_UNQUOTED(DATADIR,"$data_dir",[where to look for the data files])
|
||||
|
||||
|
||||
dnl configure libexec directory
|
||||
|
||||
libexec_dir=`echo $libexecdir | grep prefix`
|
||||
|
||||
if test -n "$libexec_dir"; then
|
||||
if test "$prefix" = "NONE"
|
||||
then
|
||||
libexec_dir="$ac_default_prefix/libexec"
|
||||
else
|
||||
libexec_dir="$prefix/libexec"
|
||||
fi
|
||||
else
|
||||
libexec_dir="$libexecdir"
|
||||
fi
|
||||
|
||||
LIBEXECDIR=$libexec_dir
|
||||
AC_SUBST(LIBEXECDIR)
|
||||
AC_DEFINE_UNQUOTED(LIBEXECDIR,"$libexec_dir",[where to look for the piler helpers])
|
||||
AC_DEFINE_UNQUOTED(VIRUS_TEMPLATE, "$my_prefix/share/clapf/template.virus", [where the virus template is])
|
||||
AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "$my_prefix/share/clapf/zombienets.regex", [where the virus template is])
|
||||
|
||||
|
||||
dnl configure dataroot directory
|
||||
|
||||
dataroot_dir=`echo $datarootdir | grep prefix`
|
||||
|
||||
if test -n "$dataroot_dir"; then
|
||||
if test "$prefix" = "NONE"
|
||||
then
|
||||
dataroot_dir="$ac_default_prefix/share"
|
||||
else
|
||||
dataroot_dir="$prefix/share"
|
||||
fi
|
||||
else
|
||||
dataroot_dir="$datarootdir"
|
||||
fi
|
||||
|
||||
|
||||
DATAROOTDIR=$dataroot_dir
|
||||
AC_SUBST(DATAROOTDIR)
|
||||
AC_DEFINE_UNQUOTED(DATAROOTDIR,"$dataroot_dir",[where to look for the share data files])
|
||||
|
||||
|
||||
|
||||
if test "$have_mysql" = "no" && test "$have_psql" = "no"; then
|
||||
echo
|
||||
echo "please specify the used database with --with-database=..."
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
dnl let us know if we are building on FreeBSD
|
||||
|
||||
if test "$os" = "FreeBSD"; then
|
||||
defs="$defs -DFREEBSD"
|
||||
antispam_libs="-lz -lm -lcrypto -lssl -liconv"
|
||||
MAKE="gmake"
|
||||
fi
|
||||
|
||||
if test "$os" = "Linux"; then
|
||||
@ -383,6 +270,11 @@ if test "$os" = "SunOS"; then
|
||||
if test -x /usr/xpg4/bin/id; then id_bin="/usr/xpg4/bin/id"; fi
|
||||
fi
|
||||
|
||||
if test `echo $os | grep -c CYGWIN` -eq 1; then
|
||||
defs="$defs -DCYGWIN"
|
||||
AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "zombienets.regex", [where the virus template is])
|
||||
fi
|
||||
|
||||
dnl whether we have antivirus support
|
||||
|
||||
if test "$have_clamd" = "yes" ; then
|
||||
@ -404,21 +296,6 @@ fi
|
||||
|
||||
echo
|
||||
|
||||
if test "$have_compat_storage_layout" = "yes"; then
|
||||
echo "support for older storage layout: yes"
|
||||
AC_DEFINE_UNQUOTED(HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT, 1, [compat storage layout support])
|
||||
fi
|
||||
|
||||
|
||||
if test "$have_mysql" = "yes"; then
|
||||
echo "database: mysql"
|
||||
fi
|
||||
|
||||
if test "$have_psql" = "yes"; then
|
||||
echo "database: psql"
|
||||
fi
|
||||
|
||||
|
||||
if test "$have_tre" = "yes"; then
|
||||
echo "tre library: yes"
|
||||
defs="$defs -DHAVE_TRE"
|
||||
@ -429,8 +306,19 @@ if test "$have_zip" = "yes"; then
|
||||
echo "zip library: yes"
|
||||
AC_DEFINE_UNQUOTED(HAVE_ZIP, 1, [libzip support])
|
||||
antispam_libs="$antispam_libs -lzip"
|
||||
fi
|
||||
|
||||
if test "$have_starttls" = "yes"; then
|
||||
echo "starttls support: yes"
|
||||
AC_DEFINE_UNQUOTED(HAVE_STARTTLS, 1, [starttls support])
|
||||
fi
|
||||
|
||||
if test "$have_tcpwrappers" = "yes"; then
|
||||
echo "tcpwrappers support: yes"
|
||||
AC_DEFINE_UNQUOTED(HAVE_LIBWRAP, 1, [tcpwrappers support])
|
||||
antispam_libs="$antispam_libs -lwrap -lnsl"
|
||||
else
|
||||
echo "zip library: no"
|
||||
echo "tcpwrappers support: no"
|
||||
fi
|
||||
|
||||
|
||||
@ -453,18 +341,16 @@ if test "$have_mysql" = "yes"; then
|
||||
fi
|
||||
|
||||
if test "$have_icc_guide" = "yes" && test "$have_mysql" = "yes"; then
|
||||
sql_libs="$sql_libs -lguide"
|
||||
mysql_libs="$mysql_libs -lguide"
|
||||
fi
|
||||
|
||||
if test "$have_psql" = "yes"; then
|
||||
defs="$defs -DNEED_PSQL"
|
||||
fi
|
||||
|
||||
if test z`which pdftotext 2>/dev/null` != "z"; then
|
||||
pdftotext=`which pdftotext`
|
||||
AC_DEFINE_UNQUOTED(HAVE_PDFTOTEXT, "$pdftotext", [path to pdftotext])
|
||||
fi
|
||||
|
||||
|
||||
if test z`which catdoc 2>/dev/null` != "z"; then
|
||||
catdoc=`which catdoc`
|
||||
AC_DEFINE_UNQUOTED(HAVE_CATDOC, "$catdoc", [path to catdoc])
|
||||
@ -495,12 +381,6 @@ if test z`which unrtf 2>/dev/null` != "z"; then
|
||||
fi
|
||||
|
||||
|
||||
if test z`which tnef 2>/dev/null` != "z"; then
|
||||
tnef=`which tnef`
|
||||
AC_DEFINE_UNQUOTED(HAVE_TNEF, "$tnef", [path to tnef])
|
||||
fi
|
||||
|
||||
|
||||
if test "$have_tweak_sent_time" = "yes"; then
|
||||
AC_DEFINE_UNQUOTED(HAVE_TWEAK_SENT_TIME, 1, [tweak sent time])
|
||||
fi
|
||||
@ -514,35 +394,25 @@ echo "catppt: $catppt"
|
||||
echo "ppthtml: $ppthtml"
|
||||
echo "xls2csv: $xls2csv"
|
||||
echo "unrtf: $unrtf"
|
||||
echo "tnef: $tnef"
|
||||
|
||||
|
||||
id -u $RUNNING_USER 2>/dev/null 1>/dev/null
|
||||
if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please create it, first with adduser..."; exit 1; fi
|
||||
|
||||
iv=`util/gen-iv.pl`
|
||||
|
||||
echo; echo
|
||||
|
||||
gcc_version="$(gcc -dumpversion)"
|
||||
extra_cflags=""
|
||||
if [[ "${gcc_version:0:1}" -gt 6 ]]; then
|
||||
extra_cflags="-Wimplicit-fallthrough=2"
|
||||
fi
|
||||
|
||||
CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wimplicit-fallthrough=2 -Wno-format-truncation -g"
|
||||
CFLAGS="$static -O2 -Wall -g"
|
||||
LIBS="$antispam_libs $sunos_libs "
|
||||
OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs"
|
||||
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o list.o parser.o parser_utils.o rules.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o pop3.o extract.o mydomains.o mysql.o $objs"
|
||||
|
||||
AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile])
|
||||
AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile test/Makefile contrib/imap/Makefile])
|
||||
AC_OUTPUT
|
||||
|
||||
echo
|
||||
echo
|
||||
echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "Did you know that piler has an enterprise edition as well?"
|
||||
echo "Check out what it can do for you at https://mailpiler.com/#features"
|
||||
echo
|
||||
echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/en/upgrade.html"
|
||||
echo
|
||||
echo
|
||||
|
||||
@ -1,103 +0,0 @@
|
||||
#!/usr/bin/php
|
||||
<?php
|
||||
|
||||
define('EXPORT_LOCK_FILE', '/var/piler/tmp/export-attachments.lock');
|
||||
|
||||
$webuidir = "";
|
||||
|
||||
$outdir = '';
|
||||
|
||||
|
||||
$opts = 'hw:d:';
|
||||
$lopts = [
|
||||
'webui:',
|
||||
'dir:',
|
||||
'help'
|
||||
];
|
||||
|
||||
|
||||
if($options = getopt($opts, $lopts)) {
|
||||
if(isset($options['webui'])) { $webuidir = $options['webui']; }
|
||||
if(isset($options['dir'])) { $outdir = $options['dir']; }
|
||||
if(isset($options['help'])) { usage(); }
|
||||
}
|
||||
|
||||
|
||||
if($webuidir == '' || $outdir == '') { usage(); }
|
||||
|
||||
|
||||
|
||||
require_once("$webuidir/config.php");
|
||||
|
||||
require(DIR_SYSTEM . "/startup.php");
|
||||
|
||||
$request = new Request();
|
||||
Registry::set("request", $request);
|
||||
|
||||
|
||||
$start = NULL;
|
||||
|
||||
$loader = new Loader();
|
||||
Registry::set('load', $loader);
|
||||
|
||||
$loader->load->model('domain/domain');
|
||||
$loader->load->model('search/search');
|
||||
$loader->load->model('search/message');
|
||||
$loader->load->model('message/attachment');
|
||||
|
||||
|
||||
$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
|
||||
Registry::set('db', $db);
|
||||
|
||||
Registry::set('auditor_user', 1);
|
||||
|
||||
openlog("export-attachments", LOG_PID, LOG_MAIL);
|
||||
|
||||
$fp = fopen(EXPORT_LOCK_FILE, "w");
|
||||
if(!flock($fp, LOCK_EX)) {
|
||||
syslog(LOG_INFO, "WARN: couldn't get a lock on " . EXPORT_LOCK_FILE);
|
||||
exit;
|
||||
}
|
||||
|
||||
|
||||
$domain = new ModelDomainDomain();
|
||||
$attachment = new ModelMessageAttachment();
|
||||
$message = new ModelSearchMessage();
|
||||
|
||||
$domains = $domain->get_mapped_domains();
|
||||
|
||||
$last_id = $attachment->get_last_attachment_id();
|
||||
$start_id = $attachment->get_checkpoint();
|
||||
|
||||
syslog(LOG_INFO, "start: $start_id, limit: $last_id");
|
||||
|
||||
|
||||
for($i=$start_id; $i<$last_id; $i++) {
|
||||
$a = $attachment->get_attachment_by_id($i);
|
||||
$m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains);
|
||||
|
||||
$attachment->dump_attachment($outdir, "out", $m['sender'], $i, $a);
|
||||
|
||||
foreach($m['rcpt'] as $rcpt) {
|
||||
$attachment->dump_attachment($outdir, "in", $rcpt, $i, $a);
|
||||
}
|
||||
|
||||
if($i % 100 == 0) { $attachment->update_checkpoint($i); }
|
||||
}
|
||||
|
||||
$attachment->update_checkpoint($i);
|
||||
|
||||
// Release lock
|
||||
flock($fp, LOCK_UN);
|
||||
fclose($fp);
|
||||
|
||||
|
||||
|
||||
function usage() {
|
||||
print "\nUsage: " . __FILE__ . "\n\n";
|
||||
print "\t--webui <path to webui directory>\n";
|
||||
print "\t--dir <basedir to write attachments>\n";
|
||||
print "\t--help\n\n";
|
||||
|
||||
exit;
|
||||
}
|
||||
@ -1,257 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
set -o nounset
|
||||
set -x
|
||||
|
||||
MY_IP="1.2.3.4"
|
||||
PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
|
||||
MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
|
||||
MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
|
||||
SERVER_ID="${SERVER_ID:-0}"
|
||||
USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
|
||||
SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
|
||||
PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"
|
||||
|
||||
MYSQL_HOSTNAME="localhost"
|
||||
MYSQL_DATABASE="piler"
|
||||
MYSQL_USERNAME="piler"
|
||||
|
||||
SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
|
||||
DOWNLOAD_URL="https://download.mailpiler.com"
|
||||
PILER_DEB="piler_1.3.12-focal-eb2b22b_amd64.deb"
|
||||
PILER_USER="piler"
|
||||
CONFIG_SITE_PHP="/etc/piler/config-site.php"
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
install_prerequisites() {
|
||||
apt-get update
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
|
||||
php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
|
||||
libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
|
||||
mariadb-server-10.3
|
||||
|
||||
wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
|
||||
tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
|
||||
wget -O /usr/local/bin/traefik "${DOWNLOAD_URL}/generic-local/traefik"
|
||||
chmod +x /usr/local/bin/traefik
|
||||
setcap cap_net_bind_service+ep /usr/local/bin/traefik
|
||||
}
|
||||
|
||||
|
||||
fix_mysql_settings() {
|
||||
cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
|
||||
[mysqld]
|
||||
|
||||
innodb_buffer_pool_size=512M
|
||||
innodb_flush_log_at_trx_commit=1
|
||||
innodb_log_buffer_size=64M
|
||||
innodb_log_file_size=64M
|
||||
innodb_read_io_threads=4
|
||||
innodb_write_io_threads=4
|
||||
innodb_log_files_in_group=2
|
||||
|
||||
innodb_file_per_table
|
||||
PILER_CNF
|
||||
}
|
||||
|
||||
start_mysql() {
|
||||
fix_mysql_settings
|
||||
|
||||
service mysql restart
|
||||
mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
|
||||
}
|
||||
|
||||
|
||||
install_piler() {
|
||||
wget "${DOWNLOAD_URL}/piler/${PILER_DEB}"
|
||||
dpkg -i "$PILER_DEB"
|
||||
rm -f "$PILER_DEB"
|
||||
|
||||
crontab -u "$PILER_USER" /usr/share/piler/piler.cron
|
||||
|
||||
rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
|
||||
ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
|
||||
|
||||
touch /var/piler/.bash_history
|
||||
chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
|
||||
}
|
||||
|
||||
|
||||
create_my_cnf() {
|
||||
local user=$1
|
||||
local password=$2
|
||||
local my_cnf=$3
|
||||
|
||||
printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
|
||||
printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
|
||||
chown $PILER_USER:$PILER_USER "$my_cnf"
|
||||
chmod 600 "$my_cnf"
|
||||
}
|
||||
|
||||
|
||||
fix_config_site_php() {
|
||||
sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
|
||||
|
||||
{
|
||||
echo "\$config['SERVER_ID'] = $SERVER_ID;"
|
||||
echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
|
||||
echo "\$config['SPHINX_VERSION'] = 331;"
|
||||
} >> "$CONFIG_SITE_PHP"
|
||||
|
||||
if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
|
||||
echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
|
||||
fi
|
||||
|
||||
echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
|
||||
}
|
||||
|
||||
|
||||
init_db() {
|
||||
sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
|
||||
/usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
|
||||
mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
|
||||
}
|
||||
|
||||
|
||||
add_systemd_services() {
|
||||
pushd /etc/systemd/system
|
||||
|
||||
ln -sf /usr/libexec/piler/piler.service .
|
||||
ln -sf /usr/libexec/piler/piler-smtp.service .
|
||||
ln -sf /usr/libexec/piler/pilersearch.service .
|
||||
|
||||
popd
|
||||
|
||||
systemctl daemon-reload
|
||||
|
||||
systemctl enable piler
|
||||
systemctl enable piler-smtp
|
||||
systemctl enable pilersearch
|
||||
systemctl enable traefik
|
||||
}
|
||||
|
||||
|
||||
fix_configs() {
|
||||
if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
|
||||
sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
|
||||
nginx -t
|
||||
nginx -s reload
|
||||
|
||||
sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
|
||||
systemctl stop nginx
|
||||
sleep 5
|
||||
systemctl start nginx
|
||||
fi
|
||||
|
||||
if [[ ! -f /etc/piler/piler.conf ]]; then
|
||||
sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
|
||||
chmod 600 /etc/piler/piler.conf
|
||||
chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
|
||||
fi
|
||||
|
||||
sed -i 's/tls_enable=0/tls_enable=1/g' /etc/piler/piler.conf
|
||||
|
||||
sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
|
||||
-e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
|
||||
-e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
|
||||
-e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
|
||||
/etc/piler/sphinx.conf
|
||||
}
|
||||
|
||||
setup_traefik() {
|
||||
wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
|
||||
|
||||
mkdir -p /usr/local/etc/traefik
|
||||
touch /usr/local/etc/traefik/acme.json
|
||||
chmod 600 /usr/local/etc/traefik/acme.json
|
||||
chown www-data:www-data /usr/local/etc/traefik/acme.json
|
||||
|
||||
cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
|
||||
log:
|
||||
level: INFO
|
||||
|
||||
entryPoints:
|
||||
web:
|
||||
address: "$MY_IP:80"
|
||||
http:
|
||||
redirections:
|
||||
entryPoint:
|
||||
to: websecure
|
||||
scheme: https
|
||||
permanent: true
|
||||
websecure:
|
||||
address: "$MY_IP:443"
|
||||
|
||||
providers:
|
||||
file:
|
||||
filename: "/usr/local/etc/traefik/traefik.yaml"
|
||||
|
||||
certificatesResolvers:
|
||||
le:
|
||||
acme:
|
||||
storage: "/usr/local/etc/traefik/acme.json"
|
||||
email: admin@$PILER_HOSTNAME
|
||||
httpChallenge:
|
||||
entryPoint: web
|
||||
|
||||
tls:
|
||||
options:
|
||||
default:
|
||||
minVersion: VersionTLS13
|
||||
|
||||
http:
|
||||
middlewares:
|
||||
piler_headers:
|
||||
headers:
|
||||
customResponseHeaders:
|
||||
Server: ""
|
||||
Strict-Transport-Security: "max-age=31536000"
|
||||
X-Content-Type-Optionsi: "nosniff"
|
||||
Referrer-Policy: "same-origin"
|
||||
routers:
|
||||
master:
|
||||
rule: "Host(\`$PILER_HOSTNAME\`)"
|
||||
service: www
|
||||
middlewares:
|
||||
- "piler_headers"
|
||||
tls:
|
||||
certResolver: le
|
||||
services:
|
||||
www:
|
||||
loadBalancer:
|
||||
servers:
|
||||
- url: "http://127.0.0.1:80/"
|
||||
TRAEFIK
|
||||
}
|
||||
|
||||
|
||||
install_prerequisites
|
||||
|
||||
start_mysql
|
||||
|
||||
install_piler
|
||||
|
||||
create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
|
||||
create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
|
||||
|
||||
fix_config_site_php
|
||||
|
||||
setup_traefik
|
||||
|
||||
add_systemd_services
|
||||
|
||||
fix_configs
|
||||
init_db
|
||||
|
||||
su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER
|
||||
|
||||
[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
|
||||
|
||||
systemctl start pilersearch
|
||||
systemctl start piler
|
||||
systemctl start piler-smtp
|
||||
systemctl start traefik
|
||||
@ -1,258 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
set -o nounset
|
||||
set -x
|
||||
|
||||
MY_IP="1.2.3.4"
|
||||
PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
|
||||
MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
|
||||
MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
|
||||
SERVER_ID="${SERVER_ID:-0}"
|
||||
USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
|
||||
SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
|
||||
PHP_FPM_SOCKET="/var/run/php/php8.1-fpm.sock"
|
||||
|
||||
MYSQL_HOSTNAME="localhost"
|
||||
MYSQL_DATABASE="piler"
|
||||
MYSQL_USERNAME="piler"
|
||||
|
||||
DOWNLOAD_URL="https://download.mailpiler.com"
|
||||
PILER_DEB="piler_1.4.4-jammy-0573c6da_amd64.deb"
|
||||
PILER_USER="piler"
|
||||
PILER_CONF="/etc/piler/piler.conf"
|
||||
CONFIG_SITE_PHP="/etc/piler/config-site.php"
|
||||
SEARCHCFG="/etc/piler/manticore.conf"
|
||||
|
||||
TRAEFIK_VERSION="v2.11.0"
|
||||
ARCH="amd64"
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
install_prerequisites() {
|
||||
apt-get update
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql \
|
||||
php8.1-fpm php8.1-zip php8.1-ldap php8.1-gd php8.1-curl php8.1-xml \
|
||||
php8.1-mbstring ca-certificates zip catdoc unrtf poppler-utils \
|
||||
nginx tnef libzip4 libtre5 libwrap0 cron libmariadb-dev python3 \
|
||||
python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
|
||||
mariadb-server-10.6
|
||||
|
||||
wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
|
||||
tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
|
||||
chown root:root traefik
|
||||
chmod 755 traefik
|
||||
mv traefik /usr/local/bin
|
||||
setcap cap_net_bind_service+ep /usr/local/bin/traefik
|
||||
|
||||
wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
|
||||
dpkg -i manticore-repo.noarch.deb && \
|
||||
apt-get update && \
|
||||
apt-get install -y manticore manticore-columnar-lib manticore-extra && \
|
||||
rm -f manticore-repo.noarch.deb
|
||||
systemctl stop manticore
|
||||
}
|
||||
|
||||
|
||||
fix_mysql_settings() {
|
||||
cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
|
||||
[mysqld]
|
||||
|
||||
innodb_buffer_pool_size=512M
|
||||
innodb_flush_log_at_trx_commit=1
|
||||
innodb_log_buffer_size=64M
|
||||
innodb_log_file_size=64M
|
||||
innodb_read_io_threads=4
|
||||
innodb_write_io_threads=4
|
||||
innodb_log_files_in_group=2
|
||||
|
||||
innodb_file_per_table
|
||||
PILER_CNF
|
||||
}
|
||||
|
||||
start_mysql() {
|
||||
fix_mysql_settings
|
||||
|
||||
service mysql restart
|
||||
mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
|
||||
}
|
||||
|
||||
|
||||
install_piler() {
|
||||
wget "https://bitbucket.org/jsuto/piler/downloads/${PILER_DEB}"
|
||||
dpkg -i "$PILER_DEB"
|
||||
rm -f "$PILER_DEB"
|
||||
|
||||
sed -i 's/.*indexer.*/###/g' /usr/share/piler/piler.cron
|
||||
crontab -u "$PILER_USER" /usr/share/piler/piler.cron
|
||||
|
||||
rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
|
||||
ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
|
||||
|
||||
touch /var/piler/.bash_history
|
||||
chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
|
||||
}
|
||||
|
||||
|
||||
create_my_cnf() {
|
||||
local user=$1
|
||||
local password=$2
|
||||
local my_cnf=$3
|
||||
|
||||
printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
|
||||
printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
|
||||
chown $PILER_USER:$PILER_USER "$my_cnf"
|
||||
chmod 600 "$my_cnf"
|
||||
}
|
||||
|
||||
|
||||
fix_config_site_php() {
|
||||
sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
|
||||
|
||||
{
|
||||
echo "\$config['SERVER_ID'] = $SERVER_ID;"
|
||||
echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
|
||||
} >> "$CONFIG_SITE_PHP"
|
||||
|
||||
if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
|
||||
echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
|
||||
fi
|
||||
|
||||
echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
|
||||
echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
|
||||
}
|
||||
|
||||
|
||||
init_db() {
|
||||
sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
|
||||
/usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
|
||||
mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
|
||||
}
|
||||
|
||||
|
||||
add_systemd_services() {
|
||||
pushd /etc/systemd/system
|
||||
|
||||
ln -sf /usr/libexec/piler/piler.service .
|
||||
ln -sf /usr/libexec/piler/piler-smtp.service .
|
||||
ln -sf /usr/libexec/piler/pilersearch.service .
|
||||
|
||||
popd
|
||||
|
||||
systemctl daemon-reload
|
||||
|
||||
systemctl enable piler
|
||||
systemctl enable piler-smtp
|
||||
systemctl enable pilersearch
|
||||
systemctl enable traefik
|
||||
}
|
||||
|
||||
|
||||
fix_configs() {
|
||||
if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
|
||||
sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
|
||||
nginx -t
|
||||
nginx -s reload
|
||||
|
||||
sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
|
||||
systemctl stop nginx
|
||||
sleep 5
|
||||
systemctl start nginx
|
||||
fi
|
||||
|
||||
if [[ ! -f "$PILER_CONF" ]]; then
|
||||
sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > "$PILER_CONF"
|
||||
chmod 600 "$PILER_CONF"
|
||||
chown $PILER_USER:$PILER_USER "$PILER_CONF"
|
||||
fi
|
||||
|
||||
sed -i 's/tls_enable=0/tls_enable=1/g' "$PILER_CONF"
|
||||
sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
|
||||
|
||||
sed -i "s/define('RT.*/define('RT', 1);/" "$SEARCHCFG"
|
||||
}
|
||||
|
||||
setup_traefik() {
|
||||
wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
|
||||
|
||||
mkdir -p /usr/local/etc/traefik
|
||||
touch /usr/local/etc/traefik/acme.json
|
||||
chmod 600 /usr/local/etc/traefik/acme.json
|
||||
chown www-data:www-data /usr/local/etc/traefik/acme.json
|
||||
|
||||
cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
|
||||
log:
|
||||
level: INFO
|
||||
|
||||
entryPoints:
|
||||
websecure:
|
||||
address: "$MY_IP:443"
|
||||
|
||||
providers:
|
||||
file:
|
||||
filename: "/usr/local/etc/traefik/traefik.yaml"
|
||||
|
||||
certificatesResolvers:
|
||||
le:
|
||||
acme:
|
||||
storage: "/usr/local/etc/traefik/acme.json"
|
||||
email: admin@$PILER_HOSTNAME
|
||||
tlsChallenge: {}
|
||||
|
||||
tls:
|
||||
options:
|
||||
default:
|
||||
minVersion: VersionTLS13
|
||||
|
||||
http:
|
||||
middlewares:
|
||||
piler_headers:
|
||||
headers:
|
||||
customResponseHeaders:
|
||||
Server: ""
|
||||
Strict-Transport-Security: "max-age=31536000"
|
||||
X-Content-Type-Optionsi: "nosniff"
|
||||
Referrer-Policy: "same-origin"
|
||||
routers:
|
||||
master:
|
||||
rule: "Host(\`$PILER_HOSTNAME\`)"
|
||||
service: www
|
||||
middlewares:
|
||||
- "piler_headers"
|
||||
tls:
|
||||
certResolver: le
|
||||
services:
|
||||
www:
|
||||
loadBalancer:
|
||||
servers:
|
||||
- url: "http://127.0.0.1:80/"
|
||||
TRAEFIK
|
||||
}
|
||||
|
||||
|
||||
install_prerequisites
|
||||
|
||||
start_mysql
|
||||
|
||||
install_piler
|
||||
|
||||
create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
|
||||
create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
|
||||
|
||||
fix_config_site_php
|
||||
|
||||
setup_traefik
|
||||
|
||||
add_systemd_services
|
||||
|
||||
fix_configs
|
||||
init_db
|
||||
|
||||
[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
|
||||
|
||||
systemctl start pilersearch
|
||||
systemctl start piler
|
||||
systemctl start piler-smtp
|
||||
systemctl start traefik
|
||||
@ -1,133 +0,0 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
|
||||
use Carp qw(verbose);
|
||||
use Sendmail::PMilter qw(:all);
|
||||
use Data::Dumper;
|
||||
use Sys::Syslog;
|
||||
use Sys::Syslog qw(:DEFAULT setlogsock);
|
||||
use Time::HiRes qw(gettimeofday);
|
||||
use Getopt::Long;
|
||||
|
||||
my $savedir = "/var/piler/imap";
|
||||
my $username = 'piler';
|
||||
my $conn = 'inet:33333@127.0.0.1';
|
||||
|
||||
my $miltername = 'pilter';
|
||||
my $priority = "mail|info";
|
||||
my $fname = '';
|
||||
my $msglen = 0;
|
||||
my $newdir = "new";
|
||||
my $curdir = "cur";
|
||||
my $newname;
|
||||
my $curname;
|
||||
my $messageid;
|
||||
my $help = 0;
|
||||
|
||||
my $opts = GetOptions(
|
||||
"conn=s" => \$conn,
|
||||
"username=s" => \$username,
|
||||
"savedir=s" => \$savedir,
|
||||
"h" => \$help,
|
||||
"help" => \$help
|
||||
);
|
||||
|
||||
if($help == 1) { die("usage: $0 --conn inet:33333\@127.0.0.1 --username piler --savedir /var/piler/imap"); }
|
||||
|
||||
chdir $savedir || die("cannot chdir to $savedir");
|
||||
|
||||
my %cbs;
|
||||
for my $cb (qw(close connect helo abort envfrom envrcpt header eoh body eom)) {
|
||||
$cbs{$cb} = sub {
|
||||
my $ctx = shift;
|
||||
my ($seconds, $microseconds);
|
||||
|
||||
###if($cb eq "connect") {
|
||||
if($cb eq "envrcpt") {
|
||||
($seconds, $microseconds) = gettimeofday;
|
||||
$fname = $seconds . "-" . $microseconds . "-" . $$ . "-" . &get_random_name;
|
||||
|
||||
###syslog $priority, "fname=$fname";
|
||||
|
||||
if(! -d $newdir) { mkdir $newdir, 0700; }
|
||||
|
||||
$newname = $newdir . "/" . $fname;
|
||||
$curname = $curdir . "/" . $fname;
|
||||
|
||||
$msglen = 0;
|
||||
$messageid = '';
|
||||
|
||||
if(!open(F, ">$newname")) { return SMFIS_TEMPFAIL; }
|
||||
}
|
||||
|
||||
elsif($cb eq "header") {
|
||||
$msglen += length ( @_[0] . ": " . @_[1] . "\n" );
|
||||
print F @_[0] . ": " . @_[1] . "\n";
|
||||
|
||||
if(@_[0] =~ /^message-id$/i) { $messageid = @_[1]; }
|
||||
}
|
||||
|
||||
elsif($cb eq "eoh") {
|
||||
print F "\n";
|
||||
}
|
||||
|
||||
elsif($cb eq "body") {
|
||||
$msglen += @_[1];
|
||||
print F @_[0];
|
||||
}
|
||||
|
||||
elsif($cb eq "eom") {
|
||||
close F;
|
||||
|
||||
if(! -d $curdir) { mkdir $curdir, 0755; }
|
||||
|
||||
rename $newname, $curname;
|
||||
|
||||
syslog $priority, "message-id=$messageid, fname=$fname, size=$msglen";
|
||||
|
||||
}
|
||||
|
||||
#if ($cb =~ /^(connect|help|envfrom|envrcpt)$/) {
|
||||
# print Dumper($ctx->{symbols})."\n";
|
||||
#}
|
||||
|
||||
SMFIS_CONTINUE;
|
||||
}
|
||||
}
|
||||
|
||||
openlog($miltername, 'pid', 'mail');
|
||||
|
||||
$< = $> = getpwnam $username;
|
||||
|
||||
syslog $priority, "$miltername starting";
|
||||
|
||||
|
||||
my $milter = new Sendmail::PMilter;
|
||||
$milter->setconn($conn);
|
||||
|
||||
$milter->register($miltername, \%cbs, SMFI_CURR_ACTS);
|
||||
|
||||
my $dispatcher = Sendmail::PMilter::prefork_dispatcher(
|
||||
max_children => 5,
|
||||
max_requests_per_child => 100,
|
||||
);
|
||||
|
||||
$milter->set_dispatcher($dispatcher);
|
||||
|
||||
$milter->main();
|
||||
|
||||
|
||||
|
||||
sub get_random_name {
|
||||
my @chars = ('A'..'Z', 0..9);
|
||||
my $s = '';
|
||||
|
||||
srand;
|
||||
|
||||
$s .= $chars[rand @chars] for 1..8;
|
||||
|
||||
return $s;
|
||||
}
|
||||
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
counter_file="pop3-position"
|
||||
COUNTER=0
|
||||
STEPS=10
|
||||
|
||||
export PATH=$PATH:/usr/bin:/usr/local/bin
|
||||
|
||||
if [ ! -f $counter_file ]; then COUNTER=1; else COUNTER=`cat $counter_file`; fi
|
||||
|
||||
|
||||
pilerimport -K pop3.youromain.com -u username -p password -s `cat $counter_file` -b $STEPS
|
||||
|
||||
|
||||
COUNTER=`expr $COUNTER + $STEPS`
|
||||
|
||||
printf "%d" "$COUNTER" > $counter_file
|
||||
@ -1,9 +0,0 @@
|
||||
You nedd proxmox with lxc
|
||||
|
||||
Fill out conf file first and provide public key
|
||||
|
||||
It automatically
|
||||
- downloads the latest debian template
|
||||
- provides a container
|
||||
- installs piler until the post installation step
|
||||
- gives a self signed certificate
|
||||
@ -1,195 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script wil create and fire up a standard debian buster lxc container on your proxmox pve.
|
||||
# The Script will look for the next free lxc number and take the next free and use it. So take
|
||||
# care that behind your last number is place for it.
|
||||
|
||||
#### SOME VARIABLES TO ADJUST ####
|
||||
|
||||
# Storage with templates
|
||||
LXC_TMP="local"
|
||||
|
||||
# Size and pool of rootfs / in GB
|
||||
SIZ_ROT="100"
|
||||
S_ROT_P="local-zfs"
|
||||
|
||||
# Size and pool of Filestorage in GB will mounted to /share
|
||||
SIZ_FIL="100"
|
||||
S_FIL_P="local-zfs"
|
||||
|
||||
#Weather or not (1 and 0) the container will createt as unpriviliged LXC
|
||||
LXC_UNP="1"
|
||||
|
||||
# Size of the RAM assigned to the LXC
|
||||
LXC_MEM="1024"
|
||||
|
||||
# Size of the SWAP assigned to the LXC
|
||||
LXC_SWA="1024"
|
||||
|
||||
# The hostname (eq. zamba1 or mailpiler1)
|
||||
LXC_HOST="zamba"
|
||||
|
||||
# The domainname (searchdomain /etc/resolf.conf & hosts)
|
||||
LXC_SDN="zmb.local"
|
||||
|
||||
# IP-address and subnet
|
||||
LXC_IP="10.10.80.20/24"
|
||||
|
||||
# Gateway
|
||||
LXC_GW="10.10.80.10"
|
||||
|
||||
# DNS-server and here shoud be your AD-DC
|
||||
LXC_DNS="10.10.80.10"
|
||||
|
||||
# Networkbridge for this machine
|
||||
LXC_BRD="vmbr80"
|
||||
|
||||
# root password - take care to delete from this file
|
||||
LXC_PWD="MYPASSWD"
|
||||
|
||||
LXC_KEY="ssh-rsa xxxxxxxx"
|
||||
|
||||
############### Zamba-Server-Section ###############
|
||||
|
||||
# Domain Entries to samba/smb.conf. Will be also uses for samba domain-provisioning when zmb-pdc will choosen.
|
||||
ZMB_REA="ZMB.LOCAL"
|
||||
ZMB_DOM="ZMB"
|
||||
|
||||
# THE Domain-Admin and passwd for zamba-install
|
||||
ZMB_ADA="Administrator"
|
||||
ZMB_APW="MYPASSWORD"
|
||||
|
||||
############### Mailpiler-Section ###############
|
||||
|
||||
# The FQDN vor the Hostname. This must be exactly the same like the LXC_HOST / LXC_SDN at section above.
|
||||
PILER_DOM="piler.zmb.rocks"
|
||||
SMARTHOST="10.10.80.20"
|
||||
PILER_VER="1.3.10"
|
||||
SPHINX_VER="3.3.1"
|
||||
PHP_VER="7.4"
|
||||
|
||||
############### Matrix-Section ###############
|
||||
|
||||
# The FQDN vor the Hostname. This should be the same like the LXC_HOST / LXC_SDN at section above.
|
||||
MRX_DOM="matrix.zmb.rocks"
|
||||
ELE_DOM="element.zmb.rocks"
|
||||
ELE_VER="v1.7.21"
|
||||
JIT_DOM="meet.zmb.rocks"
|
||||
|
||||
#################################
|
||||
|
||||
# CHeck is the newest template available, else download it.
|
||||
|
||||
DEB_LOC=$(pveam list $LXC_TMP | grep debian-10-standard | cut -d'_' -f2)
|
||||
|
||||
DEB_REP=$(pveam available --section system | grep debian-10-standard | cut -d'_' -f2)
|
||||
|
||||
if [[ $DEB_LOC == $DEB_REP ]];
|
||||
then
|
||||
echo "Newest Version of Debian 10 Standard $DEP_REP exists.";
|
||||
else
|
||||
echo "Will now download newest Debian 10 Standard $DEP_REP.";
|
||||
pveam download $LXC_TMP debian-10-standard_$DEB_REP\_amd64.tar.gz
|
||||
fi
|
||||
|
||||
# Get next free LXC-number
|
||||
LXC_LST=$( lxc-ls | egrep -o '.{1,5}$' )
|
||||
LXC_CHK=$((LXC_LST+1));
|
||||
|
||||
if [ $LXC_CHK -lt 100 ] || [ -f /etc/pve/qemu-server/$LXC_CHK.conf ]; then
|
||||
LXC_NBR=$(pvesh get /cluster/nextid);
|
||||
else
|
||||
LXC_NBR=$LXC_CHK;
|
||||
fi
|
||||
|
||||
echo "Will now create LXC Container $LXC_NBR!";
|
||||
|
||||
# Create the container
|
||||
pct create $LXC_NBR -unprivileged $LXC_UNP $LXC_TMP:vztmpl/debian-10-standard_$DEB_REP\_amd64.tar.gz -rootfs $S_ROT_P:$SIZ_ROT;
|
||||
sleep 2;
|
||||
|
||||
pct set $LXC_NBR -memory $LXC_MEM -swap $LXC_SWA -hostname $LXC_HOST \-nameserver $LXC_DNS -searchdomain $LXC_SDN -onboot 1 -timezone Europe/Berlin -net0 name=eth0,bridge=$LXC_BRD,firewall=1,gw=$LXC_GW,ip=$LXC_IP,type=veth;
|
||||
sleep 2;
|
||||
|
||||
PS3="Select the Server-Function: "
|
||||
|
||||
select opt in just_lxc zmb-standalone zmb-member zmb-pdc mailpiler matrix quit; do
|
||||
case $opt in
|
||||
just_lxc)
|
||||
lxc-start $LXC_NBR;
|
||||
sleep 5;
|
||||
# Set the root password and key
|
||||
echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
|
||||
lxc-attach -n$LXC_NBR mkdir /root/.ssh;
|
||||
echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
|
||||
lxc-attach -n$LXC_NBR service ssh restart;
|
||||
echo "Should be ready!"
|
||||
break
|
||||
;;
|
||||
zmb-standalone)
|
||||
break
|
||||
;;
|
||||
zmb-member)
|
||||
echo "Make some additions to LXC for AD-Member-Server!"
|
||||
pct set $LXC_NBR -mp0 $S_FIL_P:$SIZ_FIL,mp=/tank
|
||||
sleep 2;
|
||||
lxc-start $LXC_NBR;
|
||||
sleep 5;
|
||||
# Set the root password and key
|
||||
echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
|
||||
lxc-attach -n$LXC_NBR mkdir /root/.ssh;
|
||||
echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
|
||||
lxc-attach -n$LXC_NBR service ssh restart;
|
||||
cp /root/zmb_mem.orig /root/zmb_mem.sh
|
||||
sed -i "s|#ZMB_VAR|#ZMB_VAR\nZMB_REA='$ZMB_REA'\nZMB_DOM='$ZMB_DOM'\nZMB_ADA='$ZMB_ADA'\nZMB_APW='$ZMB_APW'|" /root/zmb_mem.sh
|
||||
pct push $LXC_NBR /root/zmb_mem.sh /root/zmb_mem.sh
|
||||
echo "Install zamba as AD-Member-Server!"
|
||||
lxc-attach -n$LXC_NBR bash /root/zmb_mem.sh
|
||||
break
|
||||
;;
|
||||
zmb-pdc)
|
||||
break
|
||||
;;
|
||||
mailpiler)
|
||||
echo "Make some additions to LXC for Mailpiler!"
|
||||
pct set $LXC_NBR -features nesting=1
|
||||
sleep 2;
|
||||
lxc-start $LXC_NBR;
|
||||
sleep 5;
|
||||
# Set the root password and key
|
||||
echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
|
||||
lxc-attach -n$LXC_NBR mkdir /root/.ssh;
|
||||
echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
|
||||
lxc-attach -n$LXC_NBR service ssh restart;
|
||||
cp /root/mailpiler.orig /root/mailpiler.sh
|
||||
sed -i "s|#PILER_VAR|#PILER_VAR\nPILER_DOM='$PILER_DOM'\nSMARTHOST='$SMARTHOST'\nPILER_VER='$PILER_VER'\nSPHINX_VER='$SPHINX_VER'\nPHP_VER='$PHP_VER'|" /root/mailpiler.sh
|
||||
pct push $LXC_NBR /root/mailpiler.sh /root/mailpiler.sh
|
||||
echo "Install Mailpiler mailarchiv!"
|
||||
lxc-attach -n$LXC_NBR bash mailpiler.sh
|
||||
break
|
||||
;;
|
||||
matrix)
|
||||
echo "Make some additions to LXC for Matrix!"
|
||||
lxc-start $LXC_NBR;
|
||||
sleep 5;
|
||||
# Set the root password and key
|
||||
echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
|
||||
lxc-attach -n$LXC_NBR mkdir /root/.ssh;
|
||||
echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
|
||||
lxc-attach -n$LXC_NBR service ssh restart;
|
||||
cp /root/matrix.orig /root/matrix.sh
|
||||
sed -i "s|#MATRIX_VAR|#Matrix_VAR\nMRX_DOM='$MRX_DOM'\nELE_DOM='$ELE_DOM'\nELE_VER='$ELE_VER'\nJIT_DOM='$JIT_DOM'|" /root/matrix.sh
|
||||
pct push $LXC_NBR /root/matrix.sh /root/matrix.sh
|
||||
echo "Install Matrix Chatserver!"
|
||||
lxc-attach -n$LXC_NBR bash matrix.sh
|
||||
break
|
||||
;;
|
||||
quit)
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Invalid option!"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
@ -1,179 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
#Variables will be filled in from the mainscript:
|
||||
|
||||
#PILER_VAR
|
||||
|
||||
|
||||
HOSTNAME=$(hostname -f)
|
||||
|
||||
echo "Ensure your Hostname is set to your Piler FQDN!"
|
||||
|
||||
echo $HOSTNAME
|
||||
|
||||
if
|
||||
[ "$HOSTNAME" != "$PILER_DOM" ]
|
||||
then
|
||||
echo "Hostname doesn't match Piler_Domain! Check install.sh, /etc/hosts, /etc/hostname." && exit
|
||||
else
|
||||
echo "Hostname matches PILER_DOMAIN, so starting installation."
|
||||
fi
|
||||
|
||||
apt install -y gpg apt-transport-https lsb-release
|
||||
|
||||
wget -q https://packages.sury.org/php/apt.gpg -O- | apt-key add -
|
||||
echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/php.list
|
||||
|
||||
apt update && apt full-upgrade -y
|
||||
|
||||
apt install -y mc sysstat build-essential libwrap0-dev libpst-dev tnef libytnef0-dev unrtf catdoc libtre-dev tre-agrep poppler-utils libzip-dev unixodbc libpq5 software-properties-common libpoppler-dev openssl libssl-dev memcached telnet nginx mariadb-server default-libmysqlclient-dev python-mysqldb gcc libwrap0 libzip4 latex2rtf latex2html catdoc tnef libpq5 zipcmp zipmerge ziptool libsodium23
|
||||
|
||||
apt update && apt install -y php$PHP_VER-{fpm,common,ldap,mysql,cli,opcache,phpdbg,gd,memcache,json,readline,zip}
|
||||
|
||||
apt purge -y postfix
|
||||
|
||||
cat > /etc/mysql/conf.d/mailpiler.conf <<EOF
|
||||
innodb_buffer_pool_size=256M
|
||||
innodb_flush_log_at_trx_commit=1
|
||||
innodb_log_buffer_size=64M
|
||||
innodb_log_file_size=16M
|
||||
query_cache_size=0
|
||||
query_cache_type=0
|
||||
query_cache_limit=2M
|
||||
EOF
|
||||
|
||||
systemctl restart mariadb
|
||||
|
||||
cd /tmp
|
||||
wget https://download.mailpiler.com/generic-local/sphinx-$SPHINX_VER-bin.tar.gz
|
||||
tar -xvzf sphinx-$SPHINX_VER-bin.tar.gz -C /
|
||||
|
||||
groupadd piler
|
||||
useradd -g piler -m -s /bin/bash -d /var/piler piler
|
||||
usermod -L piler
|
||||
chmod 755 /var/piler
|
||||
|
||||
wget https://bitbucket.org/jsuto/piler/downloads/piler-$PILER_VER.tar.gz
|
||||
tar -xvzf piler-$PILER_VER.tar.gz
|
||||
cd piler-$PILER_VER/
|
||||
./configure --localstatedir=/var --with-database=mysql --enable-tcpwrappers --enable-memcached
|
||||
make
|
||||
make install
|
||||
ldconfig
|
||||
|
||||
cp util/postinstall.sh util/postinstall.sh.bak
|
||||
sed -i "s/ SMARTHOST=.*/ SMARTHOST="\"$SMARTHOST\""/" util/postinstall.sh
|
||||
sed -i 's/ WWWGROUP=.*/ WWWGROUP="www-data"/' util/postinstall.sh
|
||||
|
||||
make postinstall
|
||||
|
||||
cp /usr/local/etc/piler/piler.conf /usr/local/etc/piler/piler.conf.bak
|
||||
sed -i "s/hostid=.*/hostid=$PILER_DOM/" /usr/local/etc/piler/piler.conf
|
||||
sed -i "s/update_counters_to_memcached=.*/update_counters_to_memcached=1/" /usr/local/etc/piler/piler.conf
|
||||
|
||||
su piler -c "indexer --all --config /usr/local/etc/piler/sphinx.conf"
|
||||
|
||||
/etc/init.d/rc.piler start
|
||||
/etc/init.d/rc.searchd start
|
||||
|
||||
update-rc.d rc.piler defaults
|
||||
update-rc.d rc.searchd defaults
|
||||
|
||||
mkdir -p /etc/nginx/ssl
|
||||
openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/piler.key -out /etc/nginx/ssl/piler.crt -subj "/CN=$PILER_DOM" -addext "subjectAltName=DNS:$PILER_DOM"
|
||||
|
||||
cd /etc/nginx/sites-available
|
||||
cp /tmp/piler-$PILER_VER/contrib/webserver/piler-nginx.conf /etc/nginx/sites-available/
|
||||
ln -s /etc/nginx/sites-available/piler-nginx.conf /etc/nginx/sites-enabled/piler-nginx.conf
|
||||
|
||||
sed -i "s|PILER_HOST|$PILER_DOM|g" /etc/nginx/sites-available/piler-nginx.conf
|
||||
sed -i "s|/var/run/php/php7.4-fpm.sock|/var/run/php/php$PHP_VER-fpm.sock|g" /etc/nginx/sites-available/piler-nginx.conf
|
||||
|
||||
sed -i "/server_name.*/a \\
|
||||
listen 443 ssl http2;\n\n\
|
||||
ssl_certificate /etc/nginx/ssl/piler.crt;\n\
|
||||
ssl_certificate_key /etc/nginx/ssl/piler.key;\n\n\
|
||||
ssl_session_timeout 1d;\n\
|
||||
ssl_session_cache shared:SSL:15m;\n\
|
||||
ssl_session_tickets off;\n\n\
|
||||
# modern configuration of Mozilla SSL configurator. Tweak to your needs.\n\
|
||||
ssl_protocols TLSv1.2 TLSv1.3;\n\
|
||||
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;\n\
|
||||
ssl_prefer_server_ciphers off;\n\n\
|
||||
add_header X-Frame-Options SAMEORIGIN;\n\
|
||||
add_header X-Content-Type-Options nosniff;" /etc/nginx/sites-available/piler-nginx.conf
|
||||
|
||||
sed -i "/^server {.*/i\
|
||||
server {\n\
|
||||
listen 80;\n\
|
||||
server_name $PILER_DOM;\n\
|
||||
server_tokens off;\n\
|
||||
# HTTP to HTTPS redirect.\n\
|
||||
return 301 https://$PILER_DOM;\n\
|
||||
}" /etc/nginx/sites-available/piler-nginx.conf
|
||||
|
||||
cp /usr/local/etc/piler/config-site.php /usr/local/etc/piler/config-site.php.bak
|
||||
sed -i "s|\$config\['SITE_URL'\] = .*|\$config\['SITE_URL'\] = 'https://$PILER_DOM/';|" /usr/local/etc/piler/config-site.php
|
||||
cat >> /usr/local/etc/piler/config-site.php <<EOF
|
||||
|
||||
// CUSTOM
|
||||
\$config['PROVIDED_BY'] = '$PILER_DOM';
|
||||
\$config['SUPPORT_LINK'] = 'https://$PILER_DOM';
|
||||
\$config['COMPATIBILITY'] = '';
|
||||
|
||||
// fancy features.
|
||||
\$config['ENABLE_INSTANT_SEARCH'] = 1;
|
||||
\$config['ENABLE_TABLE_RESIZE'] = 1;
|
||||
|
||||
\$config['ENABLE_DELETE'] = 1;
|
||||
\$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 1;
|
||||
|
||||
// general settings.
|
||||
\$config['TIMEZONE'] = 'Europe/Berlin';
|
||||
|
||||
// authentication
|
||||
// Enable authentication against an imap server
|
||||
//\$config['ENABLE_IMAP_AUTH'] = 1;
|
||||
//\$config['RESTORE_OVER_IMAP'] = 1;
|
||||
//\$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
|
||||
//\$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
|
||||
//\$config['IMAP_HOST'] = '$SMARTHOST';
|
||||
//\$config['IMAP_PORT'] = 993;
|
||||
//\$config['IMAP_SSL'] = true;
|
||||
|
||||
// authentication against an ldap directory (disabled by default)
|
||||
//\$config['ENABLE_LDAP_AUTH'] = 1;
|
||||
//\$config['LDAP_HOST'] = '$SMARTHOST';
|
||||
//\$config['LDAP_PORT'] = 389;
|
||||
//\$config['LDAP_HELPER_DN'] = 'cn=administrator,cn=users,dc=mydomain,dc=local';
|
||||
//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
|
||||
//\$config['LDAP_MAIL_ATTR'] = 'mail';
|
||||
//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
|
||||
//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
|
||||
//\$config['LDAP_BASE_DN'] = 'ou=Benutzer,dc=krs,dc=local';
|
||||
|
||||
// authentication against an Uninvention based ldap directory
|
||||
//\$config['ENABLE_LDAP_AUTH'] = 1;
|
||||
//\$config['LDAP_HOST'] = '$SMARTHOST';
|
||||
//\$config['LDAP_PORT'] = 7389;
|
||||
//\$config['LDAP_HELPER_DN'] = 'uid=ldap-search-user,cn=users,dc=mydomain,dc=local';
|
||||
//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
|
||||
//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
|
||||
//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
|
||||
//\$config['LDAP_BASE_DN'] = 'cn=users,dc=mydomain,dc=local';
|
||||
//\$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
|
||||
//\$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
|
||||
//\$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
|
||||
//\$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
|
||||
|
||||
// special settings.
|
||||
\$config['MEMCACHED_ENABLED'] = 1;
|
||||
\$config['SPHINX_STRICT_SCHEMA'] = 1; // required for Sphinx $SPHINX_VER, see https://bitbucket.org/jsuto/piler/issues/1085/sphinx-331.
|
||||
EOF
|
||||
|
||||
nginx -t && systemctl restart nginx
|
||||
|
||||
apt autoremove -y
|
||||
apt clean -y
|
||||
|
||||
|
||||
@ -1,150 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
#MATRIX_VAR
|
||||
|
||||
|
||||
MRX_PKE=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
|
||||
|
||||
ELE_DBNAME="synapse_db"
|
||||
ELE_DBUSER="synapse_user"
|
||||
ELE_DBPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
|
||||
|
||||
apt update && apt full-upgrade -y
|
||||
|
||||
apt install -y lsb-release apt-transport-https curl gpg software-properties-common net-tools nginx mc postgresql python3-psycopg2
|
||||
|
||||
wget wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/matrix-org.list
|
||||
apt update && apt install -y matrix-synapse-py3
|
||||
systemctl enable matrix-synapse
|
||||
|
||||
ss -tulpen
|
||||
|
||||
mkdir /etc/nginx/ssl
|
||||
openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/matrix.key -out /etc/nginx/ssl/matrix.crt -subj "/CN=$MRX_DOM" -addext "subjectAltName=DNS:$MRX_DOM"
|
||||
|
||||
cat > /etc/nginx/sites-available/$MRX_DOM <<EOF
|
||||
# Virtual Host configuration for example.com
|
||||
#
|
||||
# You can move that to a different file under sites-available/ and symlink that
|
||||
# to sites-enabled/ to enable it.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name $MRX_DOM;
|
||||
|
||||
return 301 https://$MRX_DOM;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
server_name $MRX_DOM;
|
||||
|
||||
ssl on;
|
||||
ssl_certificate /etc/nginx/ssl/matrix.crt;
|
||||
ssl_certificate_key /etc/nginx/ssl/matrix.key;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8008;
|
||||
proxy_set_header X-Forwarded-For \$remote_addr;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8448 ssl;
|
||||
listen [::]:8448 ssl;
|
||||
server_name $MRX_DOM;
|
||||
|
||||
ssl on;
|
||||
ssl_certificate /etc/nginx/ssl/matrix.crt;
|
||||
ssl_certificate_key /etc/nginx/ssl/matrix.key;
|
||||
|
||||
# If you don't wanna serve a site, comment this out
|
||||
root /var/www/$MRX_DOM;
|
||||
index index.html index.htm;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8008;
|
||||
proxy_set_header X-Forwarded-For \$remote_addr;
|
||||
}
|
||||
}
|
||||
|
||||
EOF
|
||||
ln -s /etc/nginx/sites-available/$MRX_DOM /etc/nginx/sites-enabled/$MRX_DOM
|
||||
|
||||
cat > /etc/nginx/sites-available/$ELE_DOM <<EOF
|
||||
# Virtual Host configuration for example.com
|
||||
#
|
||||
# You can move that to a different file under sites-available/ and symlink that
|
||||
# to sites-enabled/ to enable it.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name $ELE_DOM;
|
||||
return 301 https://$ELE_DOM;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
server_name $ELE_DOM;
|
||||
|
||||
ssl on;
|
||||
ssl_certificate /etc/nginx/ssl/matrix.crt;
|
||||
ssl_certificate_key /etc/nginx/ssl/matrix.key;
|
||||
|
||||
# If you don't wanna serve a site, comment this out
|
||||
root /var/www/$ELE_DOM/element;
|
||||
index index.html index.htm;
|
||||
}
|
||||
|
||||
EOF
|
||||
|
||||
ln -s /etc/nginx/sites-available/$ELE_DOM /etc/nginx/sites-enabled/$ELE_DOM
|
||||
|
||||
systemctl restart nginx
|
||||
|
||||
mkdir /var/www/$ELE_DOM
|
||||
cd /var/www/$ELE_DOM
|
||||
wget https://packages.riot.im/element-release-key.asc
|
||||
gpg --import element-release-key.asc
|
||||
|
||||
wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz
|
||||
wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz.asc
|
||||
gpg --verify element-$ELE_VER.tar.gz.asc
|
||||
|
||||
tar -xzvf element-$ELE_VER.tar.gz
|
||||
ln -s element-$ELE_VER element
|
||||
chown www-data:www-data -R element
|
||||
cp ./element/config.sample.json ./element/config.json
|
||||
sed -i "s|https://matrix-client.matrix.org|https://$MRX_DOM|" ./element/config.json
|
||||
sed -i "s|\"server_name\": \"matrix.org\"|\"server_name\": \"$MRX_DOM\"|" ./element/config.json
|
||||
|
||||
su postgres <<EOF
|
||||
psql -c "CREATE USER $ELE_DBUSER WITH PASSWORD '$ELE_DBPASS';"
|
||||
psql -c "CREATE DATABASE $ELE_DBNAME ENCODING 'UTF8' LC_COLLATE='C' LC_CTYPE='C' template=template0 OWNER $ELE_DBUSER;"
|
||||
echo "Postgres User '$ELE_DBUSER' and database '$ELE_DBNAME' created."
|
||||
EOF
|
||||
|
||||
cd /
|
||||
sed -i "s|#registration_shared_secret: <PRIVATE STRING>|registration_shared_secret: \"$MRX_PKE\"|" /etc/matrix-synapse/homeserver.yaml
|
||||
sed -i "s|#public_baseurl: https://example.com/|public_baseurl: https://$MRX_DOM/|" /etc/matrix-synapse/homeserver.yaml
|
||||
sed -i "s|#enable_registration: false|enable_registration: true|" /etc/matrix-synapse/homeserver.yaml
|
||||
sed -i "s|name: sqlite3|name: psycopg2|" /etc/matrix-synapse/homeserver.yaml
|
||||
sed -i "s|database: /var/lib/matrix-synapse/homeserver.db|database: $ELE_DBNAME\n user: $ELE_DBUSER\n password: $ELE_DBPASS\n host: 127.0.0.1\n cp_min: 5\n cp_max: 10|" /etc/matrix-synapse/homeserver.yaml
|
||||
|
||||
systemctl restart matrix-synapse
|
||||
|
||||
register_new_matrix_user -c /etc/matrix-synapse/homeserver.yaml http://127.0.0.1:8008
|
||||
|
||||
#curl https://download.jitsi.org/jitsi-key.gpg.key | sh -c 'gpg --dearmor > /usr/share/keyrings/jitsi-keyring.gpg'
|
||||
#echo 'deb [signed-by=/usr/share/keyrings/jitsi-keyring.gpg] https://download.jitsi.org stable/' | tee /etc/apt/sources.list.d/jitsi-stable.list > /dev/null
|
||||
|
||||
#apt update
|
||||
#apt install -y jitsi-meet
|
||||
|
||||
|
||||
|
||||
@ -1,100 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
#ZMB_VAR
|
||||
|
||||
|
||||
|
||||
apt update && apt full-upgrade -y
|
||||
echo -ne '\n' | apt install -y acl dnsutils mc samba winbind libpam-winbind libnss-winbind krb5-user krb5-config samba-dsdb-modules samba-vfs-modules
|
||||
|
||||
mv /etc/krb5.conf /etc/krb5.conf.bak
|
||||
cat > /etc/krb5.conf <<EOF
|
||||
[libdefaults]
|
||||
default_realm = $ZMB_REA
|
||||
ticket_lifetime = 600
|
||||
dns_lookup_realm = true
|
||||
dns_lookup_kdc = true
|
||||
renew_lifetime = 7d
|
||||
EOF
|
||||
|
||||
echo -e "$ZMB_APW" | kinit -V $ZMB_ADA
|
||||
klist
|
||||
|
||||
mv /etc/samba/smb.conf /etc/samba/smb.conf.bak
|
||||
cat > /etc/samba/smb.conf <<EOF
|
||||
[global]
|
||||
workgroup = $ZMB_DOM
|
||||
security = ADS
|
||||
realm = $ZMB_REA
|
||||
server string = %h server
|
||||
|
||||
vfs objects = acl_xattr shadow_copy2
|
||||
map acl inherit = Yes
|
||||
store dos attributes = Yes
|
||||
idmap config *:backend = tdb
|
||||
idmap config *:range = 3000000-4000000
|
||||
idmap config *:schema_mode = rfc2307
|
||||
|
||||
winbind refresh tickets = Yes
|
||||
winbind use default domain = Yes
|
||||
winbind separator = /
|
||||
winbind nested groups = yes
|
||||
winbind nss info = rfc2307
|
||||
|
||||
pam password change = Yes
|
||||
passwd chat = *Enter\snew\s*\spassword:* %n\n *Retype\snew\s*\spassword:* %n\n *password\supdated\ssuccessfully* .
|
||||
passwd program = /usr/bin/passwd %u
|
||||
|
||||
template homedir = /home/%U
|
||||
template shell = /bin/bash
|
||||
bind interfaces only = Yes
|
||||
interfaces = lo eth0
|
||||
log file = /var/log/samba/log.%m
|
||||
logging = syslog
|
||||
max log size = 1000
|
||||
panic action = /usr/share/samba/panic-action %d
|
||||
|
||||
load printers = No
|
||||
printcap name = /dev/null
|
||||
printing = bsd
|
||||
disable spoolss = Yes
|
||||
|
||||
allow trusted domains = No
|
||||
dns proxy = No
|
||||
shadow: snapdir = .zfs/snapshot
|
||||
shadow: sort = desc
|
||||
shadow: format = -%Y-%m-%d-%H%M
|
||||
shadow: snapprefix = ^zfs-auto-snap_\(frequent\)\{0,1\}\(hourly\)\{0,1\}\(daily\)\{0,1\}\(monthly\)\{0,1\}
|
||||
shadow: delimiter = -20
|
||||
|
||||
[share]
|
||||
comment = Main Share
|
||||
path = /tank/share
|
||||
read only = No
|
||||
create mask = 0660
|
||||
directory mask = 0770
|
||||
inherit acls = Yes
|
||||
|
||||
|
||||
|
||||
EOF
|
||||
|
||||
systemctl restart smbd
|
||||
|
||||
echo -e "$ZMB_APW" | net ads join -U $ZMB_ADA createcomputer=Computers
|
||||
sed -i "s|files systemd|files systemd winbind|g" /etc/nsswitch.conf
|
||||
sed -i "s|#WINBINDD_OPTS=|WINBINDD_OPTS=|" /etc/default/winbind
|
||||
echo -e "session optional pam_mkhomedir.so skel=/etc/skel umask=077" >> /etc/pam.d/common-session
|
||||
|
||||
systemctl restart winbind nmbd
|
||||
wbinfo -u
|
||||
wbinfo -g
|
||||
|
||||
mkdir /tank/share
|
||||
chown 'administrator':'domain users' /tank/share
|
||||
|
||||
setfacl -Rm u:administrator:rwx,g::-,o::- /tank/share
|
||||
setfacl -Rdm u:administrator:rwx,g::-,o::- /tank/share
|
||||
|
||||
systemctl restart smbd nmbd winbind
|
||||
|
||||
@ -1,13 +0,0 @@
|
||||
setsebool -P allow_httpd_mod_auth_ntlm_winbind on
|
||||
setsebool -P httpd_can_network_connect on
|
||||
setsebool -P httpd_ssi_exec on
|
||||
|
||||
setsebool -P httpd_use_nfs 1 (in case of nfs)
|
||||
|
||||
|
||||
checkmodule -M -m -o piler.mod piler.te
|
||||
semodule_package -o piler.pp -m piler.mod
|
||||
semodule -i piler.pp
|
||||
|
||||
chcon -R --type=httpd_sys_rw_content_t /var/piler/www/tmp
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
module piler 1.2;
|
||||
|
||||
require {
|
||||
type devlog_t;
|
||||
type httpd_sys_script_t;
|
||||
type httpd_t;
|
||||
type initrc_t;
|
||||
type initrc_var_run_t;
|
||||
type kernel_t;
|
||||
type public_content_t;
|
||||
type var_t;
|
||||
|
||||
class capability { kill setuid setgid sys_resource };
|
||||
class dir search;
|
||||
class file { read execute open getattr };
|
||||
class netlink_audit_socket create;
|
||||
class process { setrlimit signal };
|
||||
class sock_file write;
|
||||
class unix_dgram_socket sendto;
|
||||
|
||||
}
|
||||
|
||||
#============= httpd_sys_script_t ==============
|
||||
|
||||
allow httpd_sys_script_t var_t:file { getattr open read };
|
||||
allow httpd_sys_script_t devlog_t:sock_file write;
|
||||
allow httpd_sys_script_t httpd_t:file { getattr read open };
|
||||
allow httpd_sys_script_t httpd_t:dir search;
|
||||
allow httpd_sys_script_t initrc_t:process signal;
|
||||
allow httpd_sys_script_t initrc_var_run_t:file { getattr open read };
|
||||
allow httpd_sys_script_t kernel_t:unix_dgram_socket sendto;
|
||||
allow httpd_sys_script_t self:capability { kill setuid setgid sys_resource };
|
||||
allow httpd_sys_script_t self:netlink_audit_socket create;
|
||||
allow httpd_sys_script_t self:process setrlimit;
|
||||
|
||||
allow httpd_t var_t:file getattr;
|
||||
|
||||
|
||||
349900
contrib/smtp/dictionary.txt
349900
contrib/smtp/dictionary.txt
File diff suppressed because it is too large
Load Diff
@ -1,171 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import argparse
|
||||
import smtplib
|
||||
import random
|
||||
import string
|
||||
import quopri
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
import email.utils
|
||||
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
|
||||
"""
|
||||
check out http://blog.magiksys.net/generate-and-send-mail-with-python-tutorial
|
||||
for a real professional solution
|
||||
"""
|
||||
|
||||
eol = "\r\n"
|
||||
emails = []
|
||||
dictionary = '/'.join(__file__.split('/')[:-1]) + "/dictionary.txt"
|
||||
checkpoint_for_newline = 300
|
||||
|
||||
|
||||
def create_random_string(len=60):
|
||||
return ''.join(random.choice(string.uppercase + string.digits) for i in range(len))
|
||||
|
||||
|
||||
def create_msg(words, size=10000):
|
||||
msg_text = ""
|
||||
msg_size = 0
|
||||
new_line_checkpoint = checkpoint_for_newline
|
||||
|
||||
while msg_size < size:
|
||||
choice = random.SystemRandom().choice
|
||||
s = str(choice(words)).replace('\n', ' ')
|
||||
msg_text += s
|
||||
msg_size += len(s)
|
||||
if msg_size > new_line_checkpoint:
|
||||
new_line_checkpoint += checkpoint_for_newline
|
||||
msg_text += eol + eol
|
||||
|
||||
return msg_text
|
||||
|
||||
|
||||
def create_text_mime_part(text, charset, encoding, boundary):
|
||||
part = ""
|
||||
|
||||
part += "--" + boundary + eol
|
||||
part += "Content-Type: text/plain; charset=\"" + charset + "\"" + eol
|
||||
part += "Content-Transfer-Encoding: " + encoding + eol
|
||||
part += eol
|
||||
part += text + eol + eol
|
||||
|
||||
return part
|
||||
|
||||
|
||||
def create_headers(args, boundary):
|
||||
hdr = ""
|
||||
|
||||
hdr += "Content-Type: multipart/alternative;" + eol
|
||||
hdr += " boundary=\"" + boundary + "\"" + eol
|
||||
hdr += "MIME-Version: 1.0" + eol
|
||||
hdr += "Subject: " + args.subject + eol
|
||||
hdr += "From: " + args.sender + eol
|
||||
hdr += "To: " + ', ' . join(args.rcpt) + eol
|
||||
hdr += "Date: " + email.utils.formatdate(time.time(), localtime=True) + eol
|
||||
hdr += "Message-ID: " + "<" + create_random_string(40) + "@" + args.helo + ">" + eol
|
||||
hdr += eol + eol
|
||||
|
||||
return hdr
|
||||
|
||||
|
||||
def create_message(words, args):
|
||||
msg = ""
|
||||
|
||||
boundary = '_' + create_random_string() + '_'
|
||||
|
||||
msg += create_headers(args, boundary)
|
||||
|
||||
the_text = create_msg(words, args.msglen)
|
||||
the_text = quopri.encodestring(the_text)
|
||||
|
||||
msg += create_text_mime_part(the_text, "iso-8859-2", "quoted-printable", boundary)
|
||||
msg += "--" + boundary + eol
|
||||
|
||||
return msg
|
||||
|
||||
|
||||
def read_emails_from_dir(directory):
|
||||
for dir, dirs, files in os.walk(directory):
|
||||
for file in files:
|
||||
emails.append(os.path.join(dir, file))
|
||||
|
||||
args.count = len(emails)
|
||||
|
||||
|
||||
def get_next_email_text():
|
||||
if emails == []: return ''
|
||||
|
||||
filename = emails.pop()
|
||||
with open(filename) as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument("--helo", type=str, help="ehlo hostname", default="myhost.aaa.fu")
|
||||
parser.add_argument("-f", "--sender", type=str, help="sender email", default="sender@aaa.fu")
|
||||
parser.add_argument("-r", "--rcpt", type=str, nargs='+', help="recipient email(s)", default=["archive@aaa.fu"])
|
||||
parser.add_argument("-c", "--count", type=int, help="number of emails to send", default=1)
|
||||
parser.add_argument("-d", "--debug", type=int, help="debug level", default=0)
|
||||
parser.add_argument("--session", type=int, help="number of emails to send in one smtp transaction", default=1)
|
||||
parser.add_argument("-s", "--server", type=str, help="smtp server", required=True)
|
||||
parser.add_argument("-p", "--port", type=int, help="smtp port", default=25)
|
||||
parser.add_argument("--subject", type=str, help="subject", default="This is test subject")
|
||||
parser.add_argument("-l", "--msglen", type=int, help="message length (approx.)", default=20000)
|
||||
parser.add_argument("--starttls", help="use STARTTLS", action="store_true")
|
||||
parser.add_argument("--pem", type=str, help="pem file for starttls", default="")
|
||||
parser.add_argument("--dir", type=str, help="directory to send emails (must be eml files) from", default="")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
if args.starttls and args.pem == "":
|
||||
sys.exit("make a pem file for starttls")
|
||||
|
||||
|
||||
i = 0
|
||||
total_count = 0
|
||||
|
||||
if args.dir:
|
||||
read_emails_from_dir(args.dir)
|
||||
else:
|
||||
with open(dictionary) as f:
|
||||
words = f.readlines()
|
||||
|
||||
|
||||
while i < args.count:
|
||||
server = smtplib.SMTP(args.server, args.port, args.helo, 10)
|
||||
|
||||
if args.starttls:
|
||||
server.starttls(args.pem, args.pem)
|
||||
|
||||
k = 0
|
||||
|
||||
while i < args.count and k < args.session:
|
||||
|
||||
if args.dir:
|
||||
message = get_next_email_text()
|
||||
else:
|
||||
message = create_message(words, args)
|
||||
|
||||
server.set_debuglevel(args.debug)
|
||||
server.sendmail(args.sender, args.rcpt, message)
|
||||
|
||||
k += 1
|
||||
total_count += 1
|
||||
|
||||
if args.debug == 0:
|
||||
sys.stdout.write('%s\r' % str(total_count))
|
||||
sys.stdout.flush()
|
||||
|
||||
server.quit()
|
||||
i += 1
|
||||
|
||||
|
||||
if args.debug == 0:
|
||||
print('')
|
||||
@ -1,15 +0,0 @@
|
||||
<VirtualHost *:80>
|
||||
ServerName HOSTNAME
|
||||
|
||||
DocumentRoot "/var/piler/www"
|
||||
|
||||
<Directory /var/piler/www>
|
||||
Require all granted
|
||||
|
||||
AllowOverride all
|
||||
</Directory>
|
||||
|
||||
ErrorLog "/var/log/apache2/HOSTNAME-error_log"
|
||||
CustomLog "/var/log/apache2/HOSTNAME-access_log" common
|
||||
</VirtualHost>
|
||||
|
||||
@ -1,69 +0,0 @@
|
||||
|
||||
server {
|
||||
server_name PILER_HOST;
|
||||
|
||||
root /var/piler/www;
|
||||
|
||||
server_tokens off;
|
||||
|
||||
add_header X-XSS-Protection "1; mode=block";
|
||||
add_header X-Content-Type-Options "nosniff";
|
||||
add_header Referrer-Policy "same-origin";
|
||||
|
||||
gzip on;
|
||||
gzip_types text/plain application/xml text/css;
|
||||
gzip_vary on;
|
||||
|
||||
location / {
|
||||
index index.php index.html;
|
||||
try_files $uri $uri/ /index.php;
|
||||
}
|
||||
|
||||
#error_page 404 /404.html;
|
||||
|
||||
error_page 500 502 503 504 /50x.html;
|
||||
location = /50x.html {
|
||||
root html;
|
||||
}
|
||||
|
||||
location ~ [^/]\.php(/|$) {
|
||||
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
|
||||
|
||||
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
|
||||
if (!-f $document_root$fastcgi_script_name) {
|
||||
return 404;
|
||||
}
|
||||
|
||||
fastcgi_pass unix:/var/run/php/php8.1-fpm.sock;
|
||||
fastcgi_index index.php;
|
||||
include fastcgi_params;
|
||||
}
|
||||
|
||||
location ~* \.(ico|css|js|gif|jpe?g|png)$ {
|
||||
expires 2w;
|
||||
}
|
||||
|
||||
rewrite /search.php /index.php?route=search/search&type=simple;
|
||||
rewrite /advanced.php /index.php?route=search/search&type=advanced;
|
||||
rewrite /expert.php /index.php?route=search/search&type=expert;
|
||||
rewrite /search-helper.php /index.php?route=search/helper;
|
||||
rewrite /audit-helper.php /index.php?route=audit/helper;
|
||||
rewrite /message.php /index.php?route=message/view;
|
||||
rewrite /bulkrestore.php /index.php?route=message/bulkrestore;
|
||||
rewrite /bulkremove.php /index.php?route=message/bulkremove;
|
||||
rewrite /rejectremove.php /index.php?route=message/rejectremove;
|
||||
rewrite /bulkpdf.php /index.php?route=message/bulkpdf;
|
||||
rewrite /folders.php /index.php?route=folder/list&;
|
||||
rewrite /settings.php /index.php?route=user/settings;
|
||||
rewrite /login.php /index.php?route=login/login;
|
||||
rewrite /logout.php /index.php?route=login/logout;
|
||||
rewrite /google.php /index.php?route=login/google;
|
||||
rewrite /domain.php /index.php?route=domain/domain;
|
||||
rewrite /ldap.php /index.php?route=ldap/list;
|
||||
rewrite /customer.php /index.php?route=customer/list;
|
||||
rewrite /retention.php /index.php?route=policy/retention;
|
||||
rewrite /archiving.php /index.php?route=policy/archiving;
|
||||
rewrite /legalhold.php /index.php?route=policy/legalhold;
|
||||
rewrite /view/javascript/piler.js /js.php;
|
||||
}
|
||||
|
||||
@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o nounset
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
cppcheck -DHAVE_PDFTOTEXT -DHAVE_PPTHTML -DHAVE_TNEF -DHAVE_UNRTF -DHAVE_XLS2CSV -DHAVE_CATPPT -DHAVE_CATDOC -DHAVE_ZIP -D_GNU_SOURCE -DHAVE_DAEMON -DHAVE_TRE -DHAVE_CLAMD -DHAVE_LIBCLAMAV -DNEED_MYSQL --error-exitcode=1 --enable=all --suppressions-list=suppressions.txt --force src/ unit_tests/
|
||||
@ -1,43 +0,0 @@
|
||||
FROM ubuntu:22.04
|
||||
|
||||
ARG PACKAGE
|
||||
|
||||
LABEL description="piler ubuntu jammy image" \
|
||||
maintainer="Janos SUTO, sj@acts.hu" \
|
||||
package="${PACKAGE}"
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
DISTRO="jammy" \
|
||||
PILER_USER="piler" \
|
||||
MYSQL_DATABASE="piler"
|
||||
|
||||
COPY ${PACKAGE} /
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
|
||||
php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
|
||||
libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
|
||||
wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
|
||||
dpkg -i manticore-repo.noarch.deb && \
|
||||
rm -f manticore-repo.noarch.deb && \
|
||||
apt-get update && \
|
||||
apt-get install -y manticore manticore-columnar-lib && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
sed -i '/session required pam_loginuid.so/c\#session required pam_loginuid.so' /etc/pam.d/cron && \
|
||||
dpkg -i ${PACKAGE} && \
|
||||
touch /etc/piler/MANTICORE && \
|
||||
ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
|
||||
rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
|
||||
crontab -u $PILER_USER /usr/share/piler/piler.cron
|
||||
|
||||
VOLUME ["/etc/piler"]
|
||||
VOLUME ["/var/piler/store"]
|
||||
VOLUME ["/var/piler/manticore"]
|
||||
|
||||
EXPOSE 25 80 443
|
||||
|
||||
COPY start.sh /start.sh
|
||||
|
||||
CMD ["/start.sh"]
|
||||
@ -1,16 +0,0 @@
|
||||
## How to run piler
|
||||
|
||||
Edit the variables in docker-compose.yaml, then run
|
||||
|
||||
```
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
## How to build the image for yourself
|
||||
|
||||
Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
|
||||
and use it as the PACKAGE build argument, eg.
|
||||
|
||||
```
|
||||
docker build --build-arg PACKAGE=piler_1.4.1-jammy-c860ca67_amd64.deb -t piler:1.4.1 .
|
||||
```
|
||||
@ -1,11 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
set -o nounset
|
||||
|
||||
IMAGE_NAME="sutoj/piler:1.4.4"
|
||||
|
||||
if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
|
||||
|
||||
docker build --pull --build-arg PACKAGE="$1" -t "$IMAGE_NAME" .
|
||||
@ -1,59 +0,0 @@
|
||||
version: "3"
|
||||
services:
|
||||
mysql:
|
||||
image: mariadb:11.1.2
|
||||
container_name: mysql
|
||||
restart: unless-stopped
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
- dac_override
|
||||
- setuid
|
||||
- setgid
|
||||
environment:
|
||||
- MYSQL_DATABASE=piler
|
||||
- MYSQL_USER=piler
|
||||
- MYSQL_PASSWORD=piler123
|
||||
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
|
||||
volumes:
|
||||
- db_data:/var/lib/mysql
|
||||
piler:
|
||||
image: sutoj/piler:1.4.4
|
||||
container_name: piler
|
||||
init: true
|
||||
environment:
|
||||
- MYSQL_HOSTNAME=mysql
|
||||
- MYSQL_DATABASE=piler
|
||||
- MYSQL_USER=piler
|
||||
- MYSQL_PASSWORD=piler123
|
||||
- PILER_HOSTNAME=cust1.acts.hu
|
||||
- RT=1
|
||||
ports:
|
||||
- "25:25"
|
||||
- "80:80"
|
||||
volumes:
|
||||
- piler_etc:/etc/piler
|
||||
- piler_manticore:/var/piler/manticore
|
||||
- piler_store:/var/piler/store
|
||||
healthcheck:
|
||||
test: curl -s smtp://localhost/
|
||||
interval: "60s"
|
||||
timeout: "3s"
|
||||
start_period: "15s"
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
memory: 512M
|
||||
limits:
|
||||
memory: 512M
|
||||
|
||||
depends_on:
|
||||
- "mysql"
|
||||
|
||||
volumes:
|
||||
db_data: {}
|
||||
piler_etc: {}
|
||||
piler_manticore: {}
|
||||
piler_store: {}
|
||||
216
docker/start.sh
216
docker/start.sh
@ -1,216 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
set -o nounset
|
||||
|
||||
CONFIG_DIR="/etc/piler"
|
||||
VOLUME_DIR="/var/piler"
|
||||
PILER_CONF="${CONFIG_DIR}/piler.conf"
|
||||
PILER_KEY="${CONFIG_DIR}/piler.key"
|
||||
PILER_PEM="${CONFIG_DIR}/piler.pem"
|
||||
PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
|
||||
SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
|
||||
CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
|
||||
PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
|
||||
RT="${RT:-0}"
|
||||
|
||||
error() {
|
||||
echo "ERROR:" "$*" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
log() {
|
||||
echo "DEBUG:" "$*"
|
||||
}
|
||||
|
||||
|
||||
pre_flight_check() {
|
||||
[[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
|
||||
[[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
|
||||
[[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
|
||||
[[ -v MYSQL_USER ]] || error "Missing MYSQL_USER env variable"
|
||||
[[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
|
||||
}
|
||||
|
||||
|
||||
give_it_to_piler() {
|
||||
local f="$1"
|
||||
|
||||
[[ -f "$f" ]] || error "${f} does not exist, aborting"
|
||||
|
||||
chown "${PILER_USER}:${PILER_USER}" "$f"
|
||||
chmod 600 "$f"
|
||||
}
|
||||
|
||||
|
||||
make_certificate() {
|
||||
local f="$1"
|
||||
local crt="/tmp/1.cert"
|
||||
local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
|
||||
|
||||
log "Making an ssl certificate"
|
||||
|
||||
openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
|
||||
cat "$crt" >> "$f"
|
||||
rm -f "$crt"
|
||||
|
||||
give_it_to_piler "$f"
|
||||
}
|
||||
|
||||
|
||||
make_piler_key() {
|
||||
local f="$1"
|
||||
|
||||
log "Generating piler.key"
|
||||
|
||||
dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
|
||||
[[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
|
||||
|
||||
give_it_to_piler "$f"
|
||||
}
|
||||
|
||||
|
||||
fix_configs() {
|
||||
[[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
|
||||
[[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
|
||||
|
||||
if [[ ! -f "$PILER_NGINX_CONF" ]]; then
|
||||
log "Writing ${PILER_NGINX_CONF}"
|
||||
|
||||
cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
|
||||
sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$PILER_CONF" ]]; then
|
||||
log "Writing ${PILER_CONF}"
|
||||
|
||||
sed \
|
||||
-e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
|
||||
-e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
|
||||
-e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
|
||||
-e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
|
||||
-e "s/tls_enable=.*/tls_enable=1/g" \
|
||||
-e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
|
||||
|
||||
{
|
||||
echo "mysqlhost=${MYSQL_HOSTNAME}"
|
||||
} >> "$PILER_CONF"
|
||||
|
||||
give_it_to_piler "$PILER_CONF"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
|
||||
log "Writing ${CONFIG_SITE_PHP}"
|
||||
|
||||
cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
|
||||
|
||||
sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
|
||||
|
||||
{
|
||||
echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
|
||||
echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
|
||||
echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
|
||||
echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
|
||||
echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
|
||||
echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
|
||||
echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
|
||||
echo "\$config['ENABLE_MEMCACHED'] = 1;"
|
||||
echo "\$memcached_server = ['memcached', 11211];"
|
||||
} >> "$CONFIG_SITE_PHP"
|
||||
fi
|
||||
|
||||
sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
|
||||
-e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
|
||||
-e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
|
||||
-e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
|
||||
-i "$SPHINX_CONF"
|
||||
|
||||
# Fixes for RT index
|
||||
|
||||
if [[ $RT -eq 1 ]]; then
|
||||
sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
|
||||
if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
|
||||
echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
|
||||
fi
|
||||
|
||||
if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
|
||||
echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
|
||||
fi
|
||||
|
||||
sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
wait_until_mysql_server_is_ready() {
|
||||
while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
|
||||
|
||||
log "${MYSQL_HOSTNAME} is ready"
|
||||
}
|
||||
|
||||
|
||||
init_database() {
|
||||
local table
|
||||
local has_metadata_table=0
|
||||
|
||||
wait_until_mysql_server_is_ready
|
||||
|
||||
while read -r table; do
|
||||
if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
|
||||
done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
|
||||
|
||||
if [[ $has_metadata_table -eq 0 ]]; then
|
||||
log "no metadata table, creating tables"
|
||||
|
||||
mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
|
||||
else
|
||||
log "metadata table exists"
|
||||
fi
|
||||
|
||||
if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
|
||||
mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
create_my_cnf_files() {
|
||||
printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
|
||||
"$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
|
||||
> "$PILER_MY_CNF"
|
||||
|
||||
give_it_to_piler "$PILER_MY_CNF"
|
||||
}
|
||||
|
||||
|
||||
start_services() {
|
||||
service cron start
|
||||
service php8.1-fpm start
|
||||
service nginx start
|
||||
rsyslogd
|
||||
}
|
||||
|
||||
|
||||
start_piler() {
|
||||
if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
|
||||
log "main1.spp does not exist, creating index files"
|
||||
su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
|
||||
fi
|
||||
|
||||
# No pid file should exist for piler
|
||||
rm -f /var/run/piler/*pid
|
||||
|
||||
/etc/init.d/rc.searchd start
|
||||
/etc/init.d/rc.piler start
|
||||
}
|
||||
|
||||
|
||||
pre_flight_check
|
||||
fix_configs
|
||||
create_my_cnf_files
|
||||
init_database
|
||||
start_services
|
||||
start_piler
|
||||
|
||||
sleep infinity
|
||||
@ -16,9 +16,9 @@ localstatedir = @localstatedir@
|
||||
CC = @CC@
|
||||
CFLAGS = @CFLAGS@ @CPPFLAGS@
|
||||
DEFS = @defs@
|
||||
INCDIR = -I. -I../.. -I../../src @INCDIR@ @sql_includes@
|
||||
INCDIR = -I. -I../.. -I../../src @INCDIR@ @mysql_includes@
|
||||
LIBDIR = -L. @LIBDIR@ @LDFLAGS@ -L../../src
|
||||
LIBS = @LIBS@ @sql_libs@
|
||||
LIBS = @LIBS@ @mysql_libs@
|
||||
RUNNING_USER = @RUNNING_USER@
|
||||
RUNNING_GROUP = `@id_bin@ -gn $(RUNNING_USER)`
|
||||
|
||||
@ -26,24 +26,14 @@ INSTALL = @INSTALL@
|
||||
|
||||
all:
|
||||
sed -e 's%pidfile=.*%pidfile=$(localstatedir)/run/piler/piler.pid%g' \
|
||||
-e 's%pemfile=%pemfile=$(sysconfdir)/piler/piler.pem%g' \
|
||||
-e 's%workdir=.*%workdir=$(localstatedir)/piler/tmp%g' < $(srcdir)/example.conf | grep -v ^\; | grep '=' | sort > $(srcdir)/piler.conf
|
||||
-e 's%iv=.*%iv=@iv@%g' \
|
||||
-e 's%workdir=.*%workdir=$(localstatedir)/piler/tmp%g' < $(srcdir)/example.conf > $(srcdir)/piler.conf
|
||||
|
||||
install:
|
||||
$(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist
|
||||
sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
|
||||
sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist
|
||||
$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
|
||||
$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist
|
||||
$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php
|
||||
$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron
|
||||
sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
|
||||
sed -i -e 's%LIBEXECDIR%$(libexecdir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
|
||||
sed -i -e 's%BINDIR%$(bindir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
|
||||
sed -i -e 's%SYSCONFDIR%$(sysconfdir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
|
||||
if [ ! -f "$(DESTDIR)$(sysconfdir)/piler.conf" ]; then $(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler.conf; fi
|
||||
|
||||
clean:
|
||||
rm -f piler.conf cron.jobs sphinx.conf.dist
|
||||
rm -f piler.conf cron.jobs
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
||||
@ -1,13 +0,0 @@
|
||||
<?php
|
||||
|
||||
define('SITE_NAME_CONST', 'SITE_NAME');
|
||||
|
||||
$config[SITE_NAME_CONST] = 'HOSTNAME';
|
||||
$config['SITE_URL'] = 'http://' . $config[SITE_NAME_CONST] . '/';
|
||||
|
||||
$config['SMTP_DOMAIN'] = $config[SITE_NAME_CONST];
|
||||
$config['SMTP_FROMADDR'] = 'no-reply@' . $config[SITE_NAME_CONST];
|
||||
$config['ADMIN_EMAIL'] = 'admin@' . $config[SITE_NAME_CONST];
|
||||
|
||||
$config['DB_PASSWORD'] = 'MYSQL_PASSWORD';
|
||||
|
||||
@ -1,14 +1,36 @@
|
||||
### PILERSTART
|
||||
5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
|
||||
30 2 * * * LIBEXECDIR/piler/indexer.main.sh
|
||||
40 3 * * * LIBEXECDIR/piler/purge.sh
|
||||
*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate
|
||||
*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate
|
||||
*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
|
||||
*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
|
||||
#########################################################
|
||||
### all the cron jobs you may need for piler ###
|
||||
### be sure to review it and adjust it for your needs ###
|
||||
#########################################################
|
||||
|
||||
root's crontab:
|
||||
|
||||
### optional: query postfix queue statistics
|
||||
*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape > LOCALSTATEDIR/piler/stat/active+incoming
|
||||
*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s > LOCALSTATEDIR/piler/stat/active+incoming-sender
|
||||
*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape deferred > LOCALSTATEDIR/piler/stat/deferred
|
||||
*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s deferred > LOCALSTATEDIR/piler/stat/deferred-sender
|
||||
|
||||
|
||||
piler's crontab:
|
||||
|
||||
### mandatory
|
||||
|
||||
*/5 * * * * LC_ALL=C mpstat | tail -1 | awk '{print $11}' > /var/piler/stat/cpu.stat
|
||||
*/15 * * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate
|
||||
*/15 * * * * /usr/local/bin/indexer --quiet tag1 --rotate
|
||||
*/15 * * * * /usr/local/bin/indexer --quiet note1 --rotate
|
||||
|
||||
### optional: the same report you can see on the health page
|
||||
30 7 * * * /usr/local/libexec/piler/daily-report.php /srv/www/webui.yourdomain.com
|
||||
|
||||
### optional: populate accouting data
|
||||
30 6 * * * /usr/local/libexec/piler/generate_stats.php /srv/www/webui.yourdomain.com
|
||||
|
||||
### optional: regular AD sync
|
||||
0 8 * * * /usr/bin/php LIBEXECDIR/piler/ldap_sync.php /srv/www/webui.yourdomain.com > LOCALSTATEDIR/piler/stat/adsync.stat
|
||||
|
||||
### optional: purge aged emails
|
||||
2 0 * * * /usr/local/bin/pilerpurge
|
||||
|
||||
### optional
|
||||
###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www >/dev/null
|
||||
###*/5 * * * * LIBEXECDIR/piler/import.sh
|
||||
|
||||
### PILEREND
|
||||
|
||||
135
etc/example.conf
135
etc/example.conf
@ -22,61 +22,31 @@ username=piler
|
||||
; The default is 7 years + 2 days (=7*365+2=2557 days)
|
||||
default_retention_days=2557
|
||||
|
||||
; The initialization vector for encryption.
|
||||
; By now it has become obsolete. Don't use it for
|
||||
; new installations. However, if you used it before
|
||||
; then you must keep it as it is.
|
||||
;iv=
|
||||
; this is a 16 character long vector
|
||||
iv=****************
|
||||
|
||||
; whether to encrypt messages (1) or not (0).
|
||||
; Make sure to set this value to your needs right after installing piler,
|
||||
; and don't change it after you got the first email. Otherwise
|
||||
; you'll have half the archive encrypted, the other half unencrypted
|
||||
; which will cause problems.
|
||||
encrypt_messages=1
|
||||
|
||||
; number of worker processes, ie. the number of simultaneous smtp connections to piler.
|
||||
; This value should be the number of cpus + 1, ie. 2 for a single cpu host
|
||||
number_of_worker_processes=2
|
||||
|
||||
; max. number of parallel connections piler-smtp can handle.
|
||||
; Important! If you want to change this value, then you must first
|
||||
; stop piler-smtp, change the value, then start piler-smtp.
|
||||
; I don't suggest to go under 10 or above 1000.
|
||||
max_connections=64
|
||||
number_of_worker_processes=10
|
||||
|
||||
; number of processed emails per each piler process
|
||||
max_requests_per_child=10000
|
||||
max_requests_per_child=1000
|
||||
|
||||
; SMTP HELO identification string
|
||||
; this should be the FQDN part of the email address
|
||||
; where you copy emails, eg. archive@piler.example.com -> piler.example.com
|
||||
hostid=piler.example.com
|
||||
|
||||
; whether to process rcpt to addresses and add them to rcpt table (1) or not (0)
|
||||
process_rcpt_to_addresses=0
|
||||
hostid=mailarchiver.localhost
|
||||
|
||||
; write pid file
|
||||
pidfile=/var/run/piler/piler.pid
|
||||
|
||||
; piler will listen here
|
||||
; if you have postfix, exim, ... installed on localhost
|
||||
; then make sure to set the listen_addr parameter for piler
|
||||
listen_addr=0.0.0.0
|
||||
listen_port=25
|
||||
|
||||
clamd_socket=/tmp/clamd
|
||||
|
||||
; check for client timeout interval. Default: 20 sec
|
||||
check_for_client_timeout_interval=20
|
||||
|
||||
; smtp timeout. Default: 60 sec
|
||||
smtp_timeout=60
|
||||
|
||||
helper_timeout=20
|
||||
|
||||
; whether to run external attachment extractors (1) or not (0)
|
||||
extract_attachments=1
|
||||
session_timeout=420
|
||||
|
||||
; the 2nd parameter of the listen() system call. Please note that this is set
|
||||
; when piler starts up and you should restart piler if you change this variable.
|
||||
@ -85,12 +55,6 @@ backlog=20
|
||||
|
||||
workdir=/var/piler/tmp
|
||||
|
||||
; whether to enable writing folder_message table (1) or not (0)
|
||||
enable_folders=0
|
||||
|
||||
; discard a message if it's shorter than this value (in bytes)
|
||||
min_message_size=100
|
||||
|
||||
;
|
||||
; starttls stuff
|
||||
;
|
||||
@ -104,18 +68,11 @@ tls_enable=0
|
||||
pemfile=
|
||||
|
||||
; cipher list to use, see 'man SSL_CTX_set_cipher_list' for more details
|
||||
cipher_list=ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
|
||||
|
||||
|
||||
; set the minimum TLS protocol version for piler-smtp daemon
|
||||
;
|
||||
; Valid values: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3
|
||||
; TLSv1 and TLSv1.1 are not recommended for security reasons
|
||||
tls_min_version=TLSv1.2
|
||||
cipher_list=HIGH:MEDIUM
|
||||
|
||||
|
||||
; piler's own header to indicate previously archived messages
|
||||
piler_header_field=X-piler-id:
|
||||
piler_header_field=X-piler: piler already archived this email
|
||||
|
||||
; extra header field to treat as To:
|
||||
;
|
||||
@ -138,30 +95,9 @@ extra_to_field=X-Envelope-To:
|
||||
; to messages without message-id.
|
||||
archive_emails_not_having_message_id=0
|
||||
|
||||
; whether to archive each and every single email received (0) or
|
||||
; only those on the mydomains list (1). The default is to archive
|
||||
; everything
|
||||
archive_only_mydomains=0
|
||||
|
||||
; whether to syslog the recipients of the email in the following format:
|
||||
; 400000.....xxxxx: rcpt=recipient1@domain.com
|
||||
; 400000.....xxxxx: rcpt=recipient2@domain.com
|
||||
syslog_recipients=0
|
||||
|
||||
; minimum word length in mail body to index
|
||||
min_word_len=1
|
||||
|
||||
; whether to enable CJK (=Chinese, Japanese, and Korean) "characters".
|
||||
; the text piler can see with CJK languages may have extremely long
|
||||
; sequences without any whitespace. To prevent the parser to drop
|
||||
; these very long sequences, enable (1) this feature. By default it's
|
||||
; disabled (0).
|
||||
enable_cjk=0
|
||||
|
||||
; whether to enable the CHUNKING / BDAT feature (1) or not (0)
|
||||
; You may read about the feature at https://tools.ietf.org/html/rfc3030)
|
||||
enable_chunking=0
|
||||
|
||||
; if piler detects this line in the mail header, then it will assume
|
||||
; the message is a spam. You should include your own antispam solution's
|
||||
; specific line.
|
||||
@ -197,7 +133,7 @@ update_counters_to_memcached=0
|
||||
; hint: if you are using a mysql replicated environment and you do _not_ want
|
||||
; piler to write to the replicated database (because you do sync it some other
|
||||
; way to the master database or you are not interested in keeping the counters
|
||||
; persistantly at all), then specify a big number here, that fits to the
|
||||
; persistantly at all), then specify a big number here, that fits to the
|
||||
; "long int" size, eg. 2147483647
|
||||
memcached_to_db_interval=900
|
||||
|
||||
@ -206,69 +142,18 @@ memcached_to_db_interval=900
|
||||
; mysql stuff
|
||||
;
|
||||
|
||||
// this can be either utf8 or utf8mb4. Make sure to match the value
|
||||
// to the charset of the piler database! Also, make sure to set this
|
||||
// value in sphinx.conf
|
||||
mysqlcharset=utf8mb4
|
||||
;mysqlhost=127.0.0.1
|
||||
;mysqlport=3306
|
||||
mysqlsocket=/var/run/mysqld/mysqld.sock
|
||||
mysqlsocket=/tmp/mysql.sock
|
||||
mysqluser=piler
|
||||
mysqlpwd=verystrongpassword
|
||||
mysqldb=piler
|
||||
mysql_connect_timeout=2
|
||||
|
||||
|
||||
;
|
||||
; manticore stuff
|
||||
;
|
||||
sphxhost=127.0.0.1
|
||||
sphxport=9306
|
||||
; whether to enable (1) or not (0) the rt index. If so, then piler
|
||||
; writes the index data directly to manticore's sql API
|
||||
rtindex=0
|
||||
|
||||
; if you want to change the 'sent' time as you archive the message
|
||||
; set this in seconds. This can be a postive or negative value.
|
||||
; By default this feature is not enabled, use --tweak-sent-time
|
||||
; configure option to enable it.
|
||||
tweak_sent_time_offset=0
|
||||
|
||||
; whether to enable (1) or not (0) the extra mmap dedup test feature
|
||||
; if you change it, be sure to stop, then start piler
|
||||
mmap_dedup_test=0
|
||||
|
||||
; security header that must be present in the first mail header of
|
||||
; the message. If the security_header value is not empty, then the
|
||||
; parser checks for this header line. Unless it's found it will discard
|
||||
; the given email. Note that this feature is supposed to be a security
|
||||
; mechanism against unwanted email in the archive if limiting smtp
|
||||
; clients via an IP-address list is not feasible.
|
||||
security_header=
|
||||
|
||||
; By default the archive accepts any envelope recipient addresses.
|
||||
; If your archive's port 25 is wide open to the Internet (which it
|
||||
; shouldn't be, then spammers may find it, and fill it with spam.
|
||||
;
|
||||
; By setting this variable you may restrict the envelope address
|
||||
; to a single email address, eg. some-random-address-12345@archive.example.com
|
||||
; Then the archive will reject any other envelope recipients
|
||||
archive_address=
|
||||
|
||||
; whether to enable (1) or not (0) an smtp access list similar to
|
||||
; postfix's postscreen. Valid actions in the acl file are "permit"
|
||||
; and "reject" (without quotes). See smtp.acl.example for more.
|
||||
;
|
||||
; Important! There's an implicit default deny at the end of the
|
||||
; rules. In other words if you decide to use the acl file, then
|
||||
; everyone is not explicitly permitted is denied.
|
||||
smtp_access_list=0
|
||||
|
||||
; max message size in bytes
|
||||
; piler-smtp will reject any message that's bigger than this number
|
||||
max_message_size=50000000
|
||||
|
||||
; max memory in bytes piler-smtp uses for buffering messages
|
||||
; when this limit is exceeded, no new emails will be accepted
|
||||
; until the used memory decreases below this level
|
||||
max_smtp_memory=500000000
|
||||
|
||||
@ -1,268 +0,0 @@
|
||||
#!/usr/bin/php
|
||||
<?php
|
||||
|
||||
define('LOCALSTATEDIR', '/var');
|
||||
define('NGRAM_CONFIG', " #ngram_len = 1\n #ngram_chars = U+3000..U+2FA1F\n");
|
||||
|
||||
# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
|
||||
# The following settings contains English and some Latin extras
|
||||
define('CHARSET_TABLE', "0..9, english, _, \
|
||||
U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
|
||||
U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
|
||||
U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
|
||||
U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
|
||||
|
||||
define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
|
||||
|
||||
define('RT', 0);
|
||||
|
||||
?>
|
||||
|
||||
#
|
||||
# minimal manticore configuration suited to piler
|
||||
#
|
||||
|
||||
<?php if(RT == 0) { ?>
|
||||
|
||||
source base
|
||||
{
|
||||
type = mysql
|
||||
sql_host = MYSQL_HOSTNAME
|
||||
sql_db = MYSQL_DATABASE
|
||||
sql_user = MYSQL_USERNAME
|
||||
sql_pass = MYSQL_PASSWORD
|
||||
|
||||
sql_attr_uint = size
|
||||
sql_attr_uint = sent
|
||||
sql_attr_uint = attachments
|
||||
}
|
||||
|
||||
source delta : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query_pre = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
|
||||
sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
|
||||
|
||||
sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
|
||||
}
|
||||
|
||||
source main1 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source main2 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source main3 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source main4 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source dailydelta : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
index main1
|
||||
{
|
||||
source = main1
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/main1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
stored_fields =
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index main2
|
||||
{
|
||||
source = main2
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/main2
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
stored_fields =
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index main3
|
||||
{
|
||||
source = main3
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/main3
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
stored_fields =
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index main4
|
||||
{
|
||||
source = main4
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/main4
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
stored_fields =
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index dailydelta1
|
||||
{
|
||||
source = dailydelta
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/dailydelta1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
stored_fields =
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
killlist_target = main1:kl, main2:kl, main3:kl, main4:kl, dailydelta1:kl
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index delta1
|
||||
{
|
||||
source = delta
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/delta1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
stored_fields =
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
source tag : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
|
||||
|
||||
sql_attr_uint = iid
|
||||
sql_attr_uint = uid
|
||||
}
|
||||
|
||||
source note : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
|
||||
|
||||
sql_attr_uint = iid
|
||||
sql_attr_uint = uid
|
||||
}
|
||||
|
||||
index tag1
|
||||
{
|
||||
source = tag
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/tag1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
stored_fields =
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
|
||||
index note1
|
||||
{
|
||||
source = note
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/note1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
stored_fields =
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
<?php } else { ?>
|
||||
|
||||
index piler1
|
||||
{
|
||||
type = rt
|
||||
path = /var/piler/manticore/piler1
|
||||
rt_mem_limit = 512M
|
||||
stored_fields =
|
||||
min_word_len = 1
|
||||
min_prefix_len = 5
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
# See https://manual.manticoresearch.com/Creating_an_index/Data_types#Row-wise-and-columnar-attribute-storages
|
||||
# if you want to enable columnar storage
|
||||
# columnar_attrs = *
|
||||
rt_field = sender
|
||||
rt_field = rcpt
|
||||
rt_field = senderdomain
|
||||
rt_field = rcptdomain
|
||||
rt_field = subject
|
||||
rt_field = body
|
||||
rt_field = attachment_types
|
||||
rt_attr_bigint = arrived
|
||||
rt_attr_bigint = sent
|
||||
rt_attr_uint = size
|
||||
rt_attr_uint = direction
|
||||
rt_attr_uint = folder
|
||||
rt_attr_uint = attachments
|
||||
}
|
||||
|
||||
index tag1
|
||||
{
|
||||
type = rt
|
||||
path = /var/piler/manticore/tag1
|
||||
rt_mem_limit = 16M
|
||||
stored_fields = tag
|
||||
min_word_len = 2
|
||||
min_prefix_len = 5
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
rt_field = tag
|
||||
rt_attr_bigint = mid
|
||||
rt_attr_uint = uid
|
||||
}
|
||||
|
||||
index note1
|
||||
{
|
||||
type = rt
|
||||
path = /var/piler/manticore/note1
|
||||
rt_mem_limit = 16M
|
||||
stored_fields = note
|
||||
min_word_len = 2
|
||||
min_prefix_len = 5
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
rt_field = note
|
||||
rt_attr_bigint = mid
|
||||
rt_attr_uint = uid
|
||||
}
|
||||
|
||||
<?php } ?>
|
||||
|
||||
|
||||
searchd
|
||||
{
|
||||
listen = 127.0.0.1:9312
|
||||
listen = 127.0.0.1:9306:mysql
|
||||
listen = 127.0.0.1:9307:mysql_readonly
|
||||
log = /var/piler/manticore/manticore.log
|
||||
binlog_max_log_size = 256M
|
||||
binlog_path = /var/piler/manticore
|
||||
binlog_flush = 2
|
||||
query_log = /var/piler/manticore/query.log
|
||||
network_timeout = 5
|
||||
pid_file = /var/run/piler/searchd.pid
|
||||
seamless_rotate = 1
|
||||
preopen_tables = 1
|
||||
unlink_old = 1
|
||||
thread_stack = 512k
|
||||
<?php if(RT) { ?>
|
||||
rt_flush_period = 300
|
||||
<?php } ?>
|
||||
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
# Allow office365 servers. See the below URI for more
|
||||
# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide
|
||||
#
|
||||
# Anything is not listed below will be rejected
|
||||
#
|
||||
40.92.0.0/15 permit
|
||||
40.107.0.0/16 permit
|
||||
52.100.0.0/14 permit
|
||||
104.47.0.0/17 permit
|
||||
@ -1,36 +1,5 @@
|
||||
#!/usr/bin/php
|
||||
<?php
|
||||
|
||||
define('SPHINX_VERSION', 331); // If you have sphinx-3.3.1, then set SPHINX_VERSION to 331
|
||||
define('LOCALSTATEDIR', '@LOCALSTATEDIR@');
|
||||
define('NGRAM_CONFIG', " #ngram_len = 1\n #ngram_chars = U+3000..U+2FA1F\n");
|
||||
|
||||
# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
|
||||
# The following settings contains English and some Latin extras
|
||||
define('SPHINX_CHARSET_TABLE', "0..9, english, _, \
|
||||
U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
|
||||
U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
|
||||
U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
|
||||
U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
|
||||
|
||||
// Sphinx 3.3.1 introduced some strict rules for fulltext search column names
|
||||
// In order to comply with it you must enable SPHINX_STRICT_SCHEMA variable
|
||||
// Be sure to check out http://www.mailpiler.org/wiki/current:sphinx3 for more
|
||||
// NB: The SPHINX_STRICT_SCHEMA in sphinx.conf MUST BE THE SAME as in config.php (or in config-site.php)
|
||||
//
|
||||
define('SPHINX_STRICT_SCHEMA', 1);
|
||||
define('RT', 0);
|
||||
|
||||
if(SPHINX_STRICT_SCHEMA) {
|
||||
define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
|
||||
} else {
|
||||
define('SELECT_FIELDS', 'id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
|
||||
}
|
||||
|
||||
?>
|
||||
|
||||
#
|
||||
# minimal sphinx configuration suited to piler
|
||||
# Minimal Sphinx configuration sample (clean, simple, functional)
|
||||
#
|
||||
|
||||
source base
|
||||
@ -42,242 +11,172 @@ source base
|
||||
sql_pass = MYSQL_PASSWORD
|
||||
|
||||
sql_attr_uint = size
|
||||
sql_attr_uint = arrived
|
||||
sql_attr_uint = sent
|
||||
sql_attr_uint = direction
|
||||
sql_attr_uint = folder
|
||||
sql_attr_uint = attachments
|
||||
}
|
||||
|
||||
<?php if(!RT) { ?>
|
||||
|
||||
source delta : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query_pre = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
|
||||
sql_query_pre = SET NAMES utf8
|
||||
sql_query_pre = REPLACE INTO sph_counter SELECT 1, MAX(id) FROM sph_index
|
||||
sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
|
||||
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index \
|
||||
WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
|
||||
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
|
||||
<?php } else { ?>
|
||||
sql_query_kbatch = SELECT `id` FROM `metadata` WHERE `deleted`=1
|
||||
<?php } ?>
|
||||
}
|
||||
|
||||
source main1 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
sql_query_pre = SET NAMES utf8
|
||||
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1;
|
||||
}
|
||||
|
||||
source main2 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
sql_query_pre = SET NAMES utf8
|
||||
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1;
|
||||
}
|
||||
|
||||
source main3 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
sql_query_pre = SET NAMES utf8
|
||||
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1;
|
||||
}
|
||||
|
||||
source main4 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
sql_query_pre = SET NAMES utf8
|
||||
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1;
|
||||
}
|
||||
|
||||
source dailydelta : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
<?php } ?>
|
||||
|
||||
source tag : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
|
||||
sql_query_pre = SET NAMES utf8
|
||||
sql_query = SELECT `_id`, `id`, `uid`, `tag` FROM `tag`
|
||||
|
||||
sql_attr_uint = iid
|
||||
sql_attr_uint = id
|
||||
sql_attr_uint = uid
|
||||
|
||||
}
|
||||
|
||||
|
||||
source note : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
|
||||
sql_query_pre = SET NAMES utf8
|
||||
sql_query = SELECT `_id`, `id`, `uid`, `note` FROM `note`
|
||||
|
||||
sql_attr_uint = iid
|
||||
sql_attr_uint = id
|
||||
sql_attr_uint = uid
|
||||
|
||||
}
|
||||
|
||||
<?php if(!RT) { ?>
|
||||
|
||||
index main1
|
||||
{
|
||||
source = main1
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main1
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
docinfo = extern
|
||||
dict = keywords
|
||||
<?php } ?>
|
||||
min_prefix_len = 5
|
||||
source = main1
|
||||
path = /var/piler/sphinx/main1
|
||||
docinfo = extern
|
||||
charset_type = utf-8
|
||||
enable_star = 1
|
||||
min_prefix_len = 6
|
||||
min_word_len = 1
|
||||
charset_table = <?php print SPHINX_CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
|
||||
}
|
||||
|
||||
index main2
|
||||
{
|
||||
source = main2
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main2
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
path = /var/piler/sphinx/main2
|
||||
docinfo = extern
|
||||
dict = keywords
|
||||
<?php } ?>
|
||||
min_prefix_len = 5
|
||||
charset_type = utf-8
|
||||
enable_star = 1
|
||||
min_prefix_len = 6
|
||||
min_word_len = 1
|
||||
charset_table = <?php print SPHINX_CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index main3
|
||||
{
|
||||
source = main3
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main3
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
path = /var/piler/sphinx/main3
|
||||
docinfo = extern
|
||||
dict = keywords
|
||||
<?php } ?>
|
||||
min_prefix_len = 5
|
||||
charset_type = utf-8
|
||||
enable_star = 1
|
||||
min_prefix_len = 6
|
||||
min_word_len = 1
|
||||
charset_table = <?php print SPHINX_CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index main4
|
||||
{
|
||||
source = main4
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main4
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
path = /var/piler/sphinx/main4
|
||||
docinfo = extern
|
||||
dict = keywords
|
||||
<?php } ?>
|
||||
min_prefix_len = 5
|
||||
charset_type = utf-8
|
||||
enable_star = 1
|
||||
min_prefix_len = 6
|
||||
min_word_len = 1
|
||||
charset_table = <?php print SPHINX_CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index dailydelta1
|
||||
{
|
||||
source = dailydelta
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/sphinx/dailydelta1
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
docinfo = extern
|
||||
dict = keywords
|
||||
<?php } ?>
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print SPHINX_CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index delta1
|
||||
{
|
||||
source = delta
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/sphinx/delta1
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
path = /var/piler/sphinx/delta1
|
||||
docinfo = extern
|
||||
dict = keywords
|
||||
<?php } ?>
|
||||
min_prefix_len = 5
|
||||
charset_type = utf-8
|
||||
enable_star = 1
|
||||
min_prefix_len = 6
|
||||
min_word_len = 1
|
||||
charset_table = <?php print SPHINX_CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
<?php if(SPHINX_VERSION >= 310) { ?>
|
||||
kbatch = main1, main2, main3, main4, dailydelta1
|
||||
<?php } ?>
|
||||
|
||||
}
|
||||
|
||||
<?php } ?>
|
||||
|
||||
index tag1
|
||||
{
|
||||
source = tag
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/sphinx/tag1
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
path = /var/piler/sphinx/tag1
|
||||
docinfo = extern
|
||||
dict = keywords
|
||||
<?php } ?>
|
||||
min_prefix_len = 5
|
||||
charset_type = utf-8
|
||||
enable_star = 1
|
||||
min_prefix_len = 6
|
||||
min_word_len = 1
|
||||
charset_table = <?php print SPHINX_CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
|
||||
index note1
|
||||
{
|
||||
source = note
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/sphinx/note1
|
||||
<?php if(SPHINX_VERSION < 300) { ?>
|
||||
path = /var/piler/sphinx/note1
|
||||
docinfo = extern
|
||||
dict = keywords
|
||||
<?php } ?>
|
||||
min_prefix_len = 5
|
||||
charset_type = utf-8
|
||||
enable_star = 1
|
||||
min_prefix_len = 6
|
||||
min_word_len = 1
|
||||
charset_table = <?php print SPHINX_CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
<?php if(RT) { ?>
|
||||
index piler1
|
||||
{
|
||||
type = rt
|
||||
path = /var/piler/sphinx/piler1
|
||||
rt_mem_limit = 128M
|
||||
rt_field = sender
|
||||
rt_field = rcpt
|
||||
rt_field = senderdomain
|
||||
rt_field = rcptdomain
|
||||
rt_field = subject
|
||||
rt_field = body
|
||||
rt_field = attachment_types
|
||||
rt_attr_bigint = arrived
|
||||
rt_attr_bigint = sent
|
||||
rt_attr_uint = size
|
||||
rt_attr_uint = direction
|
||||
rt_attr_uint = folder
|
||||
rt_attr_uint = attachments
|
||||
}
|
||||
<?php } ?>
|
||||
|
||||
indexer
|
||||
{
|
||||
mem_limit = 256M
|
||||
mem_limit = 64M
|
||||
}
|
||||
|
||||
|
||||
searchd
|
||||
{
|
||||
listen = 127.0.0.1:9312
|
||||
listen = 127.0.0.1:9306:mysql41
|
||||
log = /dev/null
|
||||
binlog_path = /var/piler/sphinx
|
||||
binlog_max_log_size = 16M
|
||||
binlog_flush = 2
|
||||
<?php if(RT) { ?>
|
||||
rt_flush_period = 300
|
||||
<?php } ?>
|
||||
##query_log =
|
||||
read_timeout = 5
|
||||
max_children = 30
|
||||
pid_file = /var/run/piler/searchd.pid
|
||||
seamless_rotate = 1
|
||||
preopen_indexes = 1
|
||||
unlink_old = 1
|
||||
thread_stack = 512k
|
||||
workers = thread_pool
|
||||
listen = 127.0.0.1:9312
|
||||
listen = 127.0.0.1:9306:mysql41
|
||||
log = /dev/null
|
||||
binlog_path =
|
||||
##query_log =
|
||||
read_timeout = 5
|
||||
max_children = 30
|
||||
pid_file = /var/run/piler/searchd.pid
|
||||
max_matches = 1000
|
||||
seamless_rotate = 1
|
||||
preopen_indexes = 1
|
||||
unlink_old = 1
|
||||
workers = threads # for RT to work
|
||||
compat_sphinxql_magics = 0
|
||||
}
|
||||
|
||||
@ -28,8 +28,6 @@ all:
|
||||
|
||||
|
||||
install:
|
||||
$(INSTALL) -m 0755 $(srcdir)/rc.piler $(DESTDIR)/etc/init.d/rc.piler
|
||||
$(INSTALL) -m 0755 $(srcdir)/rc.searchd $(DESTDIR)/etc/init.d/rc.searchd
|
||||
|
||||
clean:
|
||||
rm -f rc.piler rc.searchd
|
||||
|
||||
@ -2,80 +2,49 @@
|
||||
##
|
||||
##
|
||||
|
||||
### BEGIN INIT INFO
|
||||
# Provides: piler
|
||||
# Required-Start: $remote_fs $syslog $named $network $time mysql
|
||||
# Required-Stop: $remote_fs $syslog $named $network mysql
|
||||
# Default-Start: 2 3 4 5
|
||||
# Default-Stop: 0 1 6
|
||||
# Short-Description: piler email archiver
|
||||
# Description: piler email archiver
|
||||
### END INIT INFO
|
||||
|
||||
|
||||
NAME=piler
|
||||
OPTIONS=""
|
||||
PID_FILE="$(SBINDIR/pilerconf $OPTIONS -q pidfile | cut -f2 -d=)"
|
||||
PID_NUMBER="$(test -f "$PID_FILE" && cat "$PID_FILE")"
|
||||
PILER_SMTP_PID="$(pgrep piler-smtp)"
|
||||
PID_FILE=`SBINDIR/pilerconf -q pidfile | cut -f2 -d=`
|
||||
PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
|
||||
|
||||
start() {
|
||||
echo "starting piler-smtp . . . "
|
||||
SBINDIR/piler-smtp -d
|
||||
|
||||
echo "starting $NAME . . ."
|
||||
SBINDIR/piler -d $OPTIONS
|
||||
echo "starting piler . . ."
|
||||
SBINDIR/piler -d
|
||||
}
|
||||
|
||||
stop() {
|
||||
if [ "$PID_NUMBER" != "" ]; then echo "stopping piler"; kill "$PID_NUMBER"; fi
|
||||
|
||||
if [ "$PILER_SMTP_PID" != "" ]; then echo "stopping piler-smtp"; kill "$PILER_SMTP_PID"; fi
|
||||
echo "stopping piler"
|
||||
kill ${PID_NUMBER}
|
||||
}
|
||||
|
||||
check_status(){
|
||||
if [ -f "/proc/${PID_NUMBER}/status" ]; then
|
||||
echo "piler is running, pid: ${PID_NUMBER}";
|
||||
else
|
||||
echo "piler is NOT running";
|
||||
fi
|
||||
|
||||
if [ "${PILER_SMTP_PID}" != '' ]; then
|
||||
echo "piler-smtp is running, pid: ${PILER_SMTP_PID}";
|
||||
else
|
||||
echo "piler-smtp is NOT running";
|
||||
fi
|
||||
test -f /proc/${PID_NUMBER}/status
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
start;
|
||||
;;
|
||||
start)
|
||||
start;
|
||||
;;
|
||||
|
||||
stop)
|
||||
stop;
|
||||
;;
|
||||
stop)
|
||||
stop;
|
||||
;;
|
||||
|
||||
status)
|
||||
if check_status; then
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
if check_status;
|
||||
then
|
||||
echo "${NAME} is running."
|
||||
else
|
||||
echo "${NAME} is not running."
|
||||
fi
|
||||
;;
|
||||
|
||||
restart)
|
||||
stop;
|
||||
sleep 1;
|
||||
start;
|
||||
;;
|
||||
restart)
|
||||
stop;
|
||||
sleep 1;
|
||||
start;
|
||||
;;
|
||||
|
||||
reload)
|
||||
kill -HUP "$PID_NUMBER"
|
||||
echo "reloaded"
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 start|stop|restart|reload|status"
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 start|stop|restart|status"
|
||||
esac
|
||||
|
||||
|
||||
@ -2,46 +2,18 @@
|
||||
##
|
||||
##
|
||||
|
||||
### BEGIN INIT INFO
|
||||
# Provides: pilersearch
|
||||
# Required-Start: $remote_fs $syslog $named $network $time
|
||||
# Required-Stop: $remote_fs $syslog $named $network
|
||||
# Default-Start: 2 3 4 5
|
||||
# Default-Stop: 0 1 6
|
||||
# Short-Description: sphinxsearch
|
||||
# Description: sphinxsearch
|
||||
### END INIT INFO
|
||||
|
||||
export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
|
||||
|
||||
NAME=searchd
|
||||
PID_FILE=/var/run/piler/searchd.pid
|
||||
PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE})
|
||||
CONFIG_FILE=sphinx.conf
|
||||
|
||||
if [ -f SYSCONFDIR/piler/MANTICORE ]; then
|
||||
CONFIG_FILE=manticore.conf
|
||||
fi
|
||||
|
||||
PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
|
||||
|
||||
start() {
|
||||
echo "starting searchd . . ."
|
||||
|
||||
if [ ! -d /var/run/piler ]; then
|
||||
mkdir -p /var/run/piler
|
||||
chown piler:piler /var/run/piler
|
||||
fi
|
||||
|
||||
if [ $(id -u) -eq 0 ]; then
|
||||
su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}"
|
||||
else
|
||||
searchd
|
||||
fi
|
||||
su piler -c 'BINDIR/searchd'
|
||||
}
|
||||
|
||||
stop() {
|
||||
echo "stopping searchd"
|
||||
kill "$PID_NUMBER"
|
||||
kill ${PID_NUMBER}
|
||||
}
|
||||
|
||||
check_status(){
|
||||
@ -60,11 +32,9 @@ case "$1" in
|
||||
status)
|
||||
if check_status;
|
||||
then
|
||||
echo "${NAME} is running."
|
||||
exit 0
|
||||
echo "${NAME} is running."
|
||||
else
|
||||
echo "${NAME} is not running."
|
||||
exit 1
|
||||
echo "${NAME} is not running."
|
||||
fi
|
||||
;;
|
||||
|
||||
|
||||
@ -2,30 +2,22 @@
|
||||
* piler-config.h.in, SJ
|
||||
*/
|
||||
|
||||
#define VERSION "x.x.x"
|
||||
|
||||
#define CONFDIR "/usr/local/etc"
|
||||
#define DATADIR "/usr/local/var"
|
||||
#define DATAROOTDIR "/usr/local/share"
|
||||
|
||||
#define KEYFILE CONFDIR "/piler/piler.key"
|
||||
#define LICENCE_SIGNATURE_FILE CONFDIR "/piler/piler.lic"
|
||||
|
||||
#define MESSAGE_ID_DEDUP_FILE DATAROOTDIR "/piler/deduphelper"
|
||||
#define KEYFILE CONFDIR "/piler.key"
|
||||
|
||||
#define HAVE_DAEMON 1
|
||||
|
||||
#undef TIMEOUT_BINARY
|
||||
|
||||
#undef HAVE_PDFTOTEXT
|
||||
#undef HAVE_CATDOC
|
||||
#undef HAVE_CATPPT
|
||||
#undef HAVE_XLS2CSV
|
||||
#undef HAVE_PPTHTML
|
||||
#undef HAVE_UNRTF
|
||||
#undef HAVE_TNEF
|
||||
#undef HAVE_ZIP
|
||||
|
||||
#undef HAVE_TWEAK_SENT_TIME
|
||||
#undef HAVE_STARTTLS
|
||||
#undef HAVE_LIBWRAP
|
||||
|
||||
#undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
|
||||
#undef HAVE_TWEAK_SENT_TIME
|
||||
|
||||
@ -16,11 +16,11 @@ localstatedir = @localstatedir@
|
||||
CC = @CC@
|
||||
CFLAGS = @CFLAGS@ @CPPFLAGS@
|
||||
DEFS = @defs@
|
||||
INCDIR = -I. -I.. @INCDIR@ @sql_includes@
|
||||
INCDIR = -I. -I.. @INCDIR@ @mysql_includes@
|
||||
LIBDIR = -L. @LIBDIR@ @LDFLAGS@
|
||||
LIBS = @LIBS@ @sql_libs@
|
||||
LIBS = @LIBS@ @mysql_libs@
|
||||
OBJS = @OBJS@
|
||||
SQL_OBJS = @sql_obj@
|
||||
MYSQL_OBJS = @mysql_obj@
|
||||
RUNNING_USER = @RUNNING_USER@
|
||||
RUNNING_GROUP = `@id_bin@ -gn $(RUNNING_USER)`
|
||||
|
||||
@ -33,22 +33,20 @@ MAKE = `which make`
|
||||
|
||||
INSTALL = @INSTALL@
|
||||
|
||||
all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test stats piler-smtp
|
||||
all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge reindex test
|
||||
install: install-piler
|
||||
|
||||
|
||||
piler: piler.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@ @libclamav_extra_libs@
|
||||
|
||||
libpiler.a: $(OBJS) $(SQL_OBJS)
|
||||
ar cr libpiler.a $(OBJS) $(SQL_OBJS)
|
||||
libpiler.a: $(OBJS) $(MYSQL_OBJS)
|
||||
ar cr libpiler.a $(OBJS) $(MYSQL_OBJS)
|
||||
ranlib libpiler.a
|
||||
$(CC) -shared -Wl,-soname,libpiler.so.$(LIBPILER_VERSION) -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(SQL_OBJS)
|
||||
$(CC) -shared -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(MYSQL_OBJS)
|
||||
ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so
|
||||
ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION)
|
||||
|
||||
piler-smtp: piler-smtp.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o screen.o $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerget: pilerget.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
|
||||
@ -62,6 +60,9 @@ pilerimport: pilerimport.c libpiler.a
|
||||
pilerexport: pilerexport.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerpurge: pilerpurge.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerconf: pilerconf.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
@ -71,11 +72,8 @@ reindex: reindex.c libpiler.a
|
||||
test: test.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilertest $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
|
||||
|
||||
stats: stats.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilerstats $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
|
||||
|
||||
%.o: $(srcdir)/%.c
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -c $< -o $@
|
||||
$(CC) $(CFLAGS) -fPIC $(INCDIR) $(DEFS) -c $< -o $@
|
||||
|
||||
|
||||
install-piler:
|
||||
@ -85,18 +83,16 @@ install-piler:
|
||||
(cd $(DESTDIR)$(libdir) && ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION))
|
||||
|
||||
$(INSTALL) -m 0755 piler $(DESTDIR)$(sbindir)
|
||||
$(INSTALL) -m 0755 piler-smtp $(DESTDIR)$(sbindir)
|
||||
$(INSTALL) -m 0755 pilerconf $(DESTDIR)$(sbindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerget $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pileraget $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerimport $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerpurge $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) reindex $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerstats $(DESTDIR)$(bindir)
|
||||
|
||||
clean:
|
||||
rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest pilerstats reindex piler-smtp
|
||||
rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge pilertest reindex
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
||||
103
src/archive.c
103
src/archive.c
@ -16,9 +16,6 @@
|
||||
#include <syslog.h>
|
||||
#include <openssl/blowfish.h>
|
||||
#include <openssl/evp.h>
|
||||
#if OPENSSL_VERSION_MAJOR >= 3
|
||||
#include <openssl/provider.h>
|
||||
#endif
|
||||
#include <zlib.h>
|
||||
#include <assert.h>
|
||||
#include <piler.h>
|
||||
@ -57,14 +54,11 @@ void zerr(int ret){
|
||||
|
||||
int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
|
||||
int ret, pos=0;
|
||||
unsigned have;
|
||||
z_stream strm;
|
||||
char *new_ptr;
|
||||
unsigned char out[REALLYBIGBUFSIZE];
|
||||
|
||||
/* expecting deflate with 32k window size (0x78) */
|
||||
if(len > 0 && in[0] != 0x78)
|
||||
return Z_DATA_ERROR;
|
||||
|
||||
/* allocate inflate state */
|
||||
|
||||
strm.zalloc = Z_NULL;
|
||||
@ -102,7 +96,7 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
|
||||
return ret;
|
||||
}
|
||||
|
||||
unsigned have = REALLYBIGBUFSIZE - strm.avail_out;
|
||||
have = REALLYBIGBUFSIZE - strm.avail_out;
|
||||
|
||||
/*
|
||||
* write the uncompressed result either to stdout
|
||||
@ -137,16 +131,11 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
|
||||
}
|
||||
|
||||
|
||||
int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg){
|
||||
int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg){
|
||||
int rc=0, n, olen, tlen, len, fd=-1;
|
||||
unsigned char *s=NULL, *addr=NULL, inbuf[REALLYBIGBUFSIZE];
|
||||
struct stat st;
|
||||
#if OPENSSL_VERSION_NUMBER < 0x10100000L
|
||||
EVP_CIPHER_CTX ctx;
|
||||
#else
|
||||
EVP_CIPHER_CTX *ctx=NULL;
|
||||
#endif
|
||||
int blocklen;
|
||||
|
||||
|
||||
if(filename == NULL) return 1;
|
||||
@ -165,47 +154,12 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
|
||||
return 1;
|
||||
}
|
||||
|
||||
// The new encryption scheme uses piler id starting with 5000....
|
||||
|
||||
if(cfg->encrypt_messages == 1){
|
||||
#if OPENSSL_VERSION_NUMBER < 0x10100000L
|
||||
EVP_CIPHER_CTX_init(&ctx);
|
||||
if(strstr(filename, "/5000")){
|
||||
rc = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
|
||||
} else {
|
||||
rc = EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
|
||||
}
|
||||
EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
|
||||
|
||||
if(!rc){
|
||||
syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
|
||||
goto CLEANUP;
|
||||
}
|
||||
|
||||
blocklen = EVP_CIPHER_CTX_block_size(&ctx);
|
||||
#else
|
||||
ctx = EVP_CIPHER_CTX_new();
|
||||
if(!ctx) goto CLEANUP;
|
||||
|
||||
EVP_CIPHER_CTX_init(ctx);
|
||||
if(strstr(filename, "/5000")){
|
||||
rc = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
|
||||
} else {
|
||||
#if OPENSSL_VERSION_MAJOR >= 3
|
||||
OSSL_PROVIDER_load(NULL, "legacy");
|
||||
OSSL_PROVIDER_load(NULL, "default");
|
||||
#endif
|
||||
rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
|
||||
}
|
||||
|
||||
if(!rc){
|
||||
syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
|
||||
goto CLEANUP;
|
||||
}
|
||||
|
||||
blocklen = EVP_CIPHER_CTX_block_size(ctx);
|
||||
#endif
|
||||
|
||||
len = st.st_size+blocklen;
|
||||
len = st.st_size+EVP_MAX_BLOCK_LENGTH;
|
||||
|
||||
s = malloc(len);
|
||||
|
||||
@ -218,11 +172,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
|
||||
|
||||
while((n = read(fd, inbuf, sizeof(inbuf)))){
|
||||
|
||||
#if OPENSSL_VERSION_NUMBER < 0x10100000L
|
||||
if(!EVP_DecryptUpdate(&ctx, s+tlen, &olen, inbuf, n)){
|
||||
#else
|
||||
if(!EVP_DecryptUpdate(ctx, s+tlen, &olen, inbuf, n)){
|
||||
#endif
|
||||
syslog(LOG_PRIORITY, "%s: EVP_DecryptUpdate()", filename);
|
||||
goto CLEANUP;
|
||||
}
|
||||
@ -231,24 +181,14 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
|
||||
}
|
||||
|
||||
|
||||
#if OPENSSL_VERSION_NUMBER < 0x10100000L
|
||||
if(EVP_DecryptFinal(&ctx, s + tlen, &olen) != 1){
|
||||
#else
|
||||
if(EVP_DecryptFinal(ctx, s + tlen, &olen) != 1){
|
||||
#endif
|
||||
syslog(LOG_PRIORITY, "%s: EVP_DecryptFinal()", filename);
|
||||
goto CLEANUP;
|
||||
}
|
||||
|
||||
|
||||
tlen += olen;
|
||||
|
||||
// old fileformat with static IV
|
||||
rc = inf(s, tlen, mode, buffer, dest);
|
||||
// new fileformat, starting with blocklen bytes of garbage
|
||||
if(rc != Z_OK && tlen >= blocklen){
|
||||
rc = inf(s+blocklen, tlen-blocklen, mode, buffer, dest);
|
||||
}
|
||||
}
|
||||
else {
|
||||
addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
|
||||
@ -261,45 +201,37 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
|
||||
|
||||
|
||||
CLEANUP:
|
||||
if(fd != -1) close(fd); //-V547
|
||||
if(fd != -1) close(fd);
|
||||
if(s) free(s);
|
||||
if(cfg->encrypt_messages == 1)
|
||||
#if OPENSSL_VERSION_NUMBER < 0x10100000L
|
||||
EVP_CIPHER_CTX_cleanup(&ctx);
|
||||
#else
|
||||
if(ctx) EVP_CIPHER_CTX_free(ctx);
|
||||
#endif
|
||||
if(cfg->encrypt_messages == 1) EVP_CIPHER_CTX_cleanup(&ctx);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg){
|
||||
int attachments;
|
||||
char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE];
|
||||
int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg){
|
||||
int i, attachments;
|
||||
char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE], pointer[SMALLBUFSIZE];
|
||||
struct ptr_array ptr_arr[MAX_ATTACHMENTS];
|
||||
#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
|
||||
struct stat st;
|
||||
#endif
|
||||
|
||||
|
||||
if(strlen(sdata->ttmpfile) != RND_STR_LEN){
|
||||
printf("invalid piler-id: %s\n", sdata->ttmpfile);
|
||||
return 1;
|
||||
}
|
||||
|
||||
attachments = query_attachments(sdata, &ptr_arr[0]);
|
||||
attachments = query_attachments(sdata, data, &ptr_arr[0], cfg);
|
||||
|
||||
if(attachments == -1){
|
||||
printf("problem querying the attachment of %s\n", sdata->ttmpfile);
|
||||
return 1;
|
||||
}
|
||||
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, *(sdata->ttmpfile+24), *(sdata->ttmpfile+25), *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
|
||||
#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
|
||||
if(stat(filename, &st)){
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
|
||||
}
|
||||
#endif
|
||||
|
||||
if(attachments == 0){
|
||||
retrieve_file_from_archive(filename, WRITE_TO_STDOUT, &buffer, dest, cfg);
|
||||
@ -310,8 +242,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
|
||||
if(buffer){
|
||||
saved_buffer = buffer;
|
||||
|
||||
for(int i=1; i<=attachments; i++){
|
||||
char pointer[SMALLBUFSIZE];
|
||||
for(i=1; i<=attachments; i++){
|
||||
snprintf(pointer, sizeof(pointer)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, i);
|
||||
|
||||
p = strstr(buffer, pointer);
|
||||
@ -321,13 +252,11 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
|
||||
buffer = p + strlen(pointer);
|
||||
|
||||
if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, ptr_arr[i].piler_id[24], ptr_arr[i].piler_id[25], ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
|
||||
|
||||
#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
|
||||
if(stat(filename, &st)){
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[RND_STR_LEN-6], ptr_arr[i].piler_id[RND_STR_LEN-5], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
|
||||
}
|
||||
#endif
|
||||
|
||||
retrieve_file_from_archive(filename, WRITE_TO_STDOUT, NULL, dest, cfg);
|
||||
}
|
||||
@ -346,3 +275,5 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
254
src/attachment.c
254
src/attachment.c
@ -16,58 +16,121 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg){
|
||||
int store_attachments(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
|
||||
uint64 id=0;
|
||||
int i, rc=1, found, affected_rows;
|
||||
struct sql sql, sql2;
|
||||
MYSQL_BIND bind[7];
|
||||
unsigned long len[7];
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
|
||||
if(prepare_sql_statement(sdata, &sql2, SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;
|
||||
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_attachment_table), SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_get_attachment_id_by_signature), SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;
|
||||
|
||||
|
||||
for(i=1; i<=state->n_attachments; i++){
|
||||
found = 0;
|
||||
uint64 id = 0;
|
||||
id = 0;
|
||||
|
||||
if(state->attachments[i].size > 0){
|
||||
|
||||
p_bind_init(&sql2);
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
sql2.sql[sql2.pos] = state->attachments[i].digest; sql2.type[sql2.pos] = TYPE_STRING; sql2.pos++;
|
||||
sql2.sql[sql2.pos] = (char *)&(state->attachments[i].size); sql2.type[sql2.pos] = TYPE_LONG; sql2.pos++;
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = state->attachments[i].digest;
|
||||
bind[0].is_null = 0;
|
||||
len[0] = strlen(state->attachments[i].digest); bind[0].length = &len[0];
|
||||
|
||||
if(p_exec_stmt(sdata, &sql2) == OK){
|
||||
|
||||
p_bind_init(&sql2);
|
||||
|
||||
sql2.sql[sql2.pos] = (char *)&id; sql2.type[sql2.pos] = TYPE_LONGLONG; sql2.len[sql2.pos] = sizeof(uint64); sql2.pos++;
|
||||
|
||||
p_store_results(&sql2);
|
||||
if(p_fetch_results(&sql2) == OK) found = 1;
|
||||
p_free_results(&sql2);
|
||||
if(mysql_stmt_bind_param(data->stmt_get_attachment_id_by_signature, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error for get attachment id: %s", sdata->ttmpfile, SQL_ATTACHMENT_TABLE, mysql_stmt_error(data->stmt_get_attachment_id_by_signature));
|
||||
goto NOT_FOUND;
|
||||
}
|
||||
|
||||
if(mysql_stmt_execute(data->stmt_get_attachment_id_by_signature)){
|
||||
syslog(LOG_PRIORITY, "%s get attachment id execute error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql)));
|
||||
goto NOT_FOUND;
|
||||
}
|
||||
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
bind[0].buffer = (char *)&id;
|
||||
bind[0].is_null = 0;
|
||||
bind[0].length = 0;
|
||||
|
||||
if(mysql_stmt_bind_result(data->stmt_get_attachment_id_by_signature, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_result() error: %s", sdata->ttmpfile, SQL_ATTACHMENT_TABLE, mysql_stmt_error(data->stmt_get_attachment_id_by_signature));
|
||||
goto NOT_FOUND;
|
||||
}
|
||||
|
||||
if(mysql_stmt_store_result(data->stmt_get_attachment_id_by_signature)){
|
||||
goto NOT_FOUND;
|
||||
}
|
||||
|
||||
if(!mysql_stmt_fetch(data->stmt_get_attachment_id_by_signature)){
|
||||
found = 1;
|
||||
}
|
||||
|
||||
NOT_FOUND:
|
||||
|
||||
if(found == 0){
|
||||
if(store_file(sdata, state->attachments[i].internalname, 0, cfg) == 0){
|
||||
if(store_file(sdata, state->attachments[i].internalname, 0, 0, cfg) == 0){
|
||||
syslog(LOG_PRIORITY, "%s: error storing attachment: %s", sdata->ttmpfile, state->attachments[i].internalname);
|
||||
goto CLOSE;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
p_bind_init(&sql);
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&i; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = state->attachments[i].digest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = state->attachments[i].filename; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = state->attachments[i].type; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&(state->attachments[i].size); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = sdata->ttmpfile;
|
||||
bind[0].is_null = 0;
|
||||
len[0] = strlen(sdata->ttmpfile); bind[0].length = &len[0];
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == ERR) goto CLOSE;
|
||||
bind[1].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[1].buffer = (char *)&i;
|
||||
bind[1].is_null = 0;
|
||||
bind[1].length = 0;
|
||||
|
||||
bind[2].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[2].buffer = state->attachments[i].digest;
|
||||
bind[2].is_null = 0;
|
||||
len[2] = strlen(state->attachments[i].digest); bind[2].length = &len[2];
|
||||
|
||||
bind[3].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[3].buffer = state->attachments[i].filename;
|
||||
bind[3].is_null = 0;
|
||||
len[3] = strlen(state->attachments[i].filename); bind[3].length = &len[3];
|
||||
|
||||
bind[4].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[4].buffer = state->attachments[i].type;
|
||||
bind[4].is_null = 0;
|
||||
len[4] = strlen(state->attachments[i].type); bind[4].length = &len[4];
|
||||
|
||||
bind[5].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[5].buffer = (char *)&(state->attachments[i].size);
|
||||
bind[5].is_null = 0;
|
||||
bind[5].length = 0;
|
||||
|
||||
bind[6].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
bind[6].buffer = (char *)&id;
|
||||
bind[6].is_null = 0;
|
||||
bind[6].length = 0;
|
||||
|
||||
|
||||
affected_rows = p_get_affected_rows(&sql);
|
||||
if(mysql_stmt_bind_param(data->stmt_insert_into_attachment_table, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_ATTACHMENT_TABLE, mysql_stmt_error(data->stmt_insert_into_attachment_table));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
if(mysql_stmt_execute(data->stmt_insert_into_attachment_table)){
|
||||
syslog(LOG_PRIORITY, "%s attachment sql error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql)));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
affected_rows = mysql_stmt_affected_rows(data->stmt_insert_into_attachment_table);
|
||||
if(affected_rows != 1){
|
||||
syslog(LOG_PRIORITY, "%s attachment sql error: affected rows: %d", sdata->ttmpfile, affected_rows);
|
||||
goto CLOSE;
|
||||
@ -83,70 +146,139 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
|
||||
rc = 0;
|
||||
|
||||
CLOSE:
|
||||
close_prepared_statement(&sql);
|
||||
close_prepared_statement(&sql2);
|
||||
mysql_stmt_close(data->stmt_insert_into_attachment_table);
|
||||
mysql_stmt_close(data->stmt_get_attachment_id_by_signature);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int query_attachment_pointers(struct session_data *sdata, uint64 ptr, char *piler_id, int *id){
|
||||
int query_attachment_pointers(struct session_data *sdata, struct __data *data, uint64 ptr, char *piler_id, int *id, struct __config *cfg){
|
||||
int rc=0;
|
||||
struct sql sql;
|
||||
MYSQL_BIND bind[2];
|
||||
my_bool is_null[2];
|
||||
unsigned long len=0;
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) return rc;
|
||||
|
||||
p_bind_init(&sql);
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_get_attachment_pointer), SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) goto ENDE;
|
||||
|
||||
sql.sql[sql.pos] = (char *)&ptr; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK){
|
||||
bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
bind[0].buffer = (char *)&ptr;
|
||||
bind[0].is_null = 0;
|
||||
len = sizeof(uint64); bind[0].length = &len;
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = piler_id; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
|
||||
|
||||
p_store_results(&sql);
|
||||
|
||||
if(p_fetch_results(&sql) == OK) rc = 1;
|
||||
p_free_results(&sql);
|
||||
if(mysql_stmt_bind_param(data->stmt_get_attachment_pointer, bind)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
|
||||
if(mysql_stmt_execute(data->stmt_get_attachment_pointer)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = piler_id;
|
||||
bind[0].buffer_length = RND_STR_LEN;
|
||||
bind[0].is_null = &is_null[0];
|
||||
bind[0].length = &len;
|
||||
|
||||
bind[1].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[1].buffer = (char *)id;
|
||||
bind[1].is_null = 0;
|
||||
bind[1].length = 0;
|
||||
|
||||
|
||||
if(mysql_stmt_bind_result(data->stmt_get_attachment_pointer, bind)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
if(mysql_stmt_store_result(data->stmt_get_attachment_pointer)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
if(!mysql_stmt_fetch(data->stmt_get_attachment_pointer)){
|
||||
if(is_null[0] == 0){
|
||||
rc = 1;
|
||||
}
|
||||
}
|
||||
|
||||
CLOSE:
|
||||
mysql_stmt_close(data->stmt_get_attachment_pointer);
|
||||
|
||||
ENDE:
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr){
|
||||
int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr, struct __config *cfg){
|
||||
int i, rc, id, attachments=0;
|
||||
uint64 ptr;
|
||||
struct sql sql;
|
||||
MYSQL_BIND bind[2];
|
||||
my_bool is_null[2];
|
||||
unsigned long len=0;
|
||||
|
||||
|
||||
for(i=0; i<MAX_ATTACHMENTS; i++) memset((char*)&ptr_arr[i], 0, sizeof(struct ptr_array));
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) return attachments;
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_query_attachment), SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) goto ENDE;
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == ERR) goto CLOSE;
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = sdata->ttmpfile;
|
||||
bind[0].is_null = 0;
|
||||
len = strlen(sdata->ttmpfile); bind[0].length = &len;
|
||||
|
||||
p_bind_init(&sql);
|
||||
if(mysql_stmt_bind_param(data->stmt_query_attachment, bind)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&ptr; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
|
||||
p_store_results(&sql);
|
||||
if(mysql_stmt_execute(data->stmt_query_attachment)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
while(p_fetch_results(&sql) == OK){
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
bind[0].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[0].buffer = (char *)&id;
|
||||
bind[0].is_null = &is_null[0];
|
||||
bind[0].length = 0;
|
||||
|
||||
bind[1].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
bind[1].buffer = (char *)&ptr;
|
||||
bind[1].is_null = &is_null[1];
|
||||
bind[1].length = 0;
|
||||
|
||||
|
||||
|
||||
if(mysql_stmt_bind_result(data->stmt_query_attachment, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_result() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(data->stmt_query_attachment));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
if(mysql_stmt_store_result(data->stmt_query_attachment)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_store_result() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(data->stmt_query_attachment));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
while(!mysql_stmt_fetch(data->stmt_query_attachment)){
|
||||
|
||||
if(id > 0 && id < MAX_ATTACHMENTS){
|
||||
if(ptr > 0){
|
||||
ptr_arr[id].ptr = ptr;
|
||||
rc = query_attachment_pointers(sdata, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
|
||||
rc = query_attachment_pointers(sdata, data, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id), cfg);
|
||||
if(!rc){
|
||||
attachments = -1;
|
||||
goto CLOSE;
|
||||
@ -161,10 +293,12 @@ int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr){
|
||||
}
|
||||
}
|
||||
|
||||
p_free_results(&sql);
|
||||
|
||||
CLOSE:
|
||||
close_prepared_statement(&sql);
|
||||
mysql_stmt_close(data->stmt_query_attachment);
|
||||
|
||||
ENDE:
|
||||
|
||||
return attachments;
|
||||
}
|
||||
|
||||
|
||||
|
||||
56
src/av.h
56
src/av.h
@ -20,6 +20,60 @@
|
||||
#define CLAMD_RESP_INFECTED "FOUND"
|
||||
#define CLAMD_RESP_ERROR "ERROR"
|
||||
|
||||
int clamd_scan(char *tmpfile, struct config *cfg);
|
||||
int clamd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
|
||||
int clamd_net_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
|
||||
|
||||
// Dr.Web stuff
|
||||
|
||||
#define DRWEB_RESP_VIRUS 0x20
|
||||
#define DRWEB_VIRUS_HAS_FOUND_MESSAGE "Virus has been found in message. See drwebd.log for details"
|
||||
|
||||
int drweb_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
|
||||
|
||||
|
||||
// avast! stuff
|
||||
|
||||
#define AVAST_READY "220"
|
||||
#define AVAST_CMD_QUIT "QUIT\r\n"
|
||||
|
||||
#define AVAST_RESP_OK "200"
|
||||
#define AVAST_RESP_ENGINE_ERROR "451"
|
||||
#define AVAST_RESP_SYNTAX_ERROR "501"
|
||||
|
||||
#define AVAST_RESP_CLEAN "[+]"
|
||||
#define AVAST_RESP_INFECTED "[L]"
|
||||
|
||||
int avast_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
|
||||
int avast_cmd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
|
||||
|
||||
// Kaspersky stuff
|
||||
|
||||
#define KAV_CMD_QUIT "QUIT\r\n"
|
||||
|
||||
#define KAV_READY "201 "
|
||||
#define KAV_RESP_CLEAN "220 File is clean"
|
||||
#define KAV_RESP_INFECTED "230 File is infected"
|
||||
#define KAV_RESP_INFECTED_NAME "322-"
|
||||
#define KAV_RESP_NOT_FOUND "525 File not found"
|
||||
|
||||
int kav_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
|
||||
|
||||
// avg stuff
|
||||
|
||||
#define AVG_READY "220"
|
||||
#define AVG_CMD_QUIT "QUIT\r\n"
|
||||
|
||||
#define AVG_RESP_OK "200"
|
||||
#define AVG_RESP_VIRUS "403"
|
||||
#define AVG_RESP_NOT_FOUND "404"
|
||||
#define AVG_RESP_ERROR "501"
|
||||
|
||||
#define AVG_NOT_FOUND 404
|
||||
|
||||
int avg_scan(char *tmpdir, char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
|
||||
|
||||
|
||||
int moveMessageToQuarantine(struct session_data *sdata, struct __config *cfg);
|
||||
void sendNotificationToPostmaster(struct session_data *sdata, char *rcpttoemail, char *fromemail, char *virusinfo, char *avengine, struct __config *cfg);
|
||||
|
||||
#endif /* _AV_H */
|
||||
|
||||
40
src/avir.c
40
src/avir.c
@ -10,12 +10,46 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int do_av_check(char *filename, struct config *cfg){
|
||||
int do_av_check(struct session_data *sdata, char *rcpttoemail, char *virusinfo, struct __data *data, struct __config *cfg){
|
||||
int rav = AVIR_OK;
|
||||
char avengine[SMALLBUFSIZE];
|
||||
|
||||
if(clamd_scan(filename, cfg) == AV_VIRUS) rav = AVIR_VIRUS;
|
||||
if(sdata->need_scan == 0) return rav;
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: done virus scanning", filename);
|
||||
memset(avengine, 0, SMALLBUFSIZE);
|
||||
|
||||
#ifdef HAVE_LIBCLAMAV
|
||||
const char *virname;
|
||||
unsigned int options=0;
|
||||
|
||||
options = CL_SCAN_STDOPT | CL_SCAN_ARCHIVE | CL_SCAN_MAIL | CL_SCAN_OLE2;
|
||||
|
||||
if(cfg->use_libclamav_block_max_feature == 1) options |= CL_SCAN_BLOCKMAX;
|
||||
|
||||
if(cfg->clamav_block_encrypted_archives == 1) options |= CL_SCAN_BLOCKENCRYPTED;
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to pass to libclamav", sdata->ttmpfile);
|
||||
|
||||
if(cl_scanfile(sdata->ttmpfile, &virname, NULL, data->engine, options) == CL_VIRUS){
|
||||
memset(virusinfo, 0, SMALLBUFSIZE);
|
||||
strncpy(virusinfo, virname, SMALLBUFSIZE-1);
|
||||
rav = AVIR_VIRUS;
|
||||
snprintf(avengine, SMALLBUFSIZE-1, "libClamAV");
|
||||
}
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: virus info: '%s'", sdata->ttmpfile, virname);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_CLAMD
|
||||
if(strlen(cfg->clamd_addr) > 3 && cfg->clamd_port > 0){
|
||||
if(clamd_net_scan(sdata->ttmpfile, avengine, virusinfo, cfg) == AV_VIRUS) rav = AVIR_VIRUS;
|
||||
} else {
|
||||
if(clamd_scan(sdata->ttmpfile, avengine, virusinfo, cfg) == AV_VIRUS) rav = AVIR_VIRUS;
|
||||
}
|
||||
#endif
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: done virus scanning", sdata->ttmpfile);
|
||||
|
||||
return rav;
|
||||
}
|
||||
|
||||
|
||||
79
src/base64.c
Normal file
79
src/base64.c
Normal file
@ -0,0 +1,79 @@
|
||||
/*
|
||||
* base64.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
|
||||
char base64_value(char c){
|
||||
static const char *base64_table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
if((int)c > 63) return '=';
|
||||
|
||||
return base64_table[(int)c];
|
||||
}
|
||||
|
||||
|
||||
void base64_encode_block(unsigned char *in, int inlen, char *out){
|
||||
char a, b, c, d, fragment;
|
||||
|
||||
sprintf(out, "====");
|
||||
|
||||
if(inlen <= 0) return;
|
||||
|
||||
fragment = *in & 0x3;
|
||||
|
||||
a = *in >> 2;
|
||||
|
||||
out[0] = base64_value(a);
|
||||
|
||||
b = fragment << 4;
|
||||
|
||||
if(inlen > 1)
|
||||
b += *(in+1) >> 4;
|
||||
|
||||
out[1] = base64_value(b);
|
||||
|
||||
if(inlen == 1) return;
|
||||
|
||||
|
||||
c = *(in+1) & 0xf;
|
||||
c = c << 2;
|
||||
|
||||
if(inlen > 2){
|
||||
fragment = *(in+2) & 0xfc;
|
||||
c += fragment >> 6;
|
||||
|
||||
d = *(in+2) & 0x3f;
|
||||
out[3] = base64_value(d);
|
||||
}
|
||||
|
||||
out[2] = base64_value(c);
|
||||
}
|
||||
|
||||
|
||||
void base64_encode(unsigned char *in, int inlen, char *out, int outlen){
|
||||
int i=0, j, pos=0;
|
||||
unsigned char buf[3];
|
||||
|
||||
memset(buf, 0, 3);
|
||||
memset(out, 0, outlen);
|
||||
|
||||
for(j=0; j<inlen; j++){
|
||||
|
||||
if(i == 3){
|
||||
base64_encode_block(buf, 3, &out[pos]); pos += 4;
|
||||
|
||||
memset(buf, 0, 3);
|
||||
|
||||
i = 0;
|
||||
}
|
||||
|
||||
buf[i] = *(in+j);
|
||||
i++;
|
||||
}
|
||||
|
||||
base64_encode_block(buf, i, &out[pos]);
|
||||
}
|
||||
|
||||
117
src/bdat.c
117
src/bdat.c
@ -1,117 +0,0 @@
|
||||
/*
|
||||
* bdat.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <strings.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <signal.h>
|
||||
#include <syslog.h>
|
||||
#include <time.h>
|
||||
#include <openssl/ssl.h>
|
||||
#include <openssl/err.h>
|
||||
#include <piler.h>
|
||||
#include <smtp.h>
|
||||
|
||||
|
||||
void reset_bdat_counters(struct smtp_session *session){
|
||||
session->bdat_bytes_to_read = 0;
|
||||
session->bad = 0;
|
||||
}
|
||||
|
||||
|
||||
void get_bdat_size_to_read(struct smtp_session *session){
|
||||
char *p;
|
||||
|
||||
session->bdat_bytes_to_read = 0;
|
||||
|
||||
session->protocol_state = SMTP_STATE_BDAT;
|
||||
|
||||
p = strcasestr(session->buf, " LAST");
|
||||
if(p){
|
||||
*p = '\0';
|
||||
}
|
||||
|
||||
// determine the size to be read
|
||||
|
||||
p = strchr(session->buf, ' ');
|
||||
if(p){
|
||||
session->bdat_bytes_to_read = atoi(p);
|
||||
if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);
|
||||
}
|
||||
|
||||
if(!p || session->bdat_bytes_to_read <= 0){
|
||||
session->bdat_bytes_to_read = 0;
|
||||
syslog(LOG_INFO, "%s: ERROR: malformed BDAT command", session->ttmpfile);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
|
||||
if(readlen <= 0) return;
|
||||
|
||||
if(session->fd == -1){
|
||||
session->fd = open(session->ttmpfile, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP);
|
||||
if(session->fd == -1){
|
||||
syslog(LOG_PRIORITY, "%s: %s", ERR_OPEN_TMP_FILE, session->ttmpfile);
|
||||
}
|
||||
|
||||
if(cfg->process_rcpt_to_addresses == 1) write_envelope_addresses(session, cfg);
|
||||
}
|
||||
|
||||
session->bdat_bytes_to_read -= readlen;
|
||||
|
||||
if(session->fd != -1){
|
||||
if(write(session->fd, readbuf, readlen) != -1){
|
||||
session->tot_len += readlen;
|
||||
|
||||
if(session->cfg->verbosity >= _LOG_EXTREME) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
|
||||
}
|
||||
else syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
|
||||
}
|
||||
|
||||
|
||||
if(session->bdat_bytes_to_read < 0){
|
||||
// malformed data from client: we got more data then had been told in BDAT argument
|
||||
syslog(LOG_PRIORITY, "ERROR: invalid BDAT data. Expected %d, got %d bytes", session->bdat_bytes_to_read + readlen, readlen);
|
||||
session->bad = 1;
|
||||
|
||||
close(session->fd);
|
||||
unlink(session->ttmpfile);
|
||||
|
||||
session->fd = -1;
|
||||
}
|
||||
|
||||
|
||||
// If there's nothing more to read, then send response to smtp client
|
||||
|
||||
if(session->bdat_bytes_to_read <= 0){
|
||||
|
||||
if(session->fd == -1){
|
||||
send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
|
||||
}
|
||||
else {
|
||||
fsync(session->fd);
|
||||
close(session->fd);
|
||||
|
||||
session->fd = -1;
|
||||
|
||||
move_email(session);
|
||||
|
||||
char buf[SMALLBUFSIZE];
|
||||
snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
|
||||
send_smtp_response(session, buf);
|
||||
syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket);
|
||||
}
|
||||
|
||||
// technically we are not in the PERIOD state, but it's good enough
|
||||
// to quit the BDAT processing state
|
||||
session->protocol_state = SMTP_STATE_PERIOD;
|
||||
}
|
||||
}
|
||||
186
src/cfg.c
186
src/cfg.c
@ -17,16 +17,30 @@ int string_parser(char *src, char *target, int limit){
|
||||
return 0;
|
||||
};
|
||||
|
||||
int int_parser(char *src, int *target){
|
||||
int multi_line_string_parser(char *src, char *target, int limit){
|
||||
if(strlen(src) > 0 && strlen(target) + strlen(src) + 3 < limit){
|
||||
strncat(target, src, limit);
|
||||
strncat(target, "\r\n", limit);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
};
|
||||
|
||||
int int_parser(char *src, int *target, int limit){
|
||||
*target = strtol(src, (char **) NULL, 10);
|
||||
|
||||
return 0;
|
||||
};
|
||||
|
||||
int uint64_parser(char *src, uint64 *target){
|
||||
*target = strtoull(src, (char**)NULL, 10);
|
||||
|
||||
int float_parser(char *src, float *target, int limit){
|
||||
*target = strtof(src, (char **) NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct _parse_rule {
|
||||
char *name;
|
||||
@ -44,73 +58,51 @@ struct _parse_rule {
|
||||
|
||||
struct _parse_rule config_parse_rules[] =
|
||||
{
|
||||
{ "archive_address", "string", (void*) string_parser, offsetof(struct config, archive_address), "", MAXVAL-1},
|
||||
{ "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct config, archive_emails_not_having_message_id), "0", sizeof(int)},
|
||||
{ "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct config, archive_only_mydomains), "0", sizeof(int)},
|
||||
{ "backlog", "integer", (void*) int_parser, offsetof(struct config, backlog), "20", sizeof(int)},
|
||||
{ "check_for_client_timeout_interval", "integer", (void*) int_parser, offsetof(struct config, check_for_client_timeout_interval), "20", sizeof(int)},
|
||||
{ "cipher_list", "string", (void*) string_parser, offsetof(struct config, cipher_list), "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS", MAXVAL-1},
|
||||
{ "clamd_addr", "string", (void*) string_parser, offsetof(struct config, clamd_addr), "", MAXVAL-1},
|
||||
{ "clamd_port", "integer", (void*) int_parser, offsetof(struct config, clamd_port), "0", sizeof(int)},
|
||||
{ "clamd_socket", "string", (void*) string_parser, offsetof(struct config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
|
||||
{ "debug", "integer", (void*) int_parser, offsetof(struct config, debug), "0", sizeof(int)},
|
||||
{ "default_retention_days", "integer", (void*) int_parser, offsetof(struct config, default_retention_days), "2557", sizeof(int)},
|
||||
{ "enable_chunking", "integer", (void*) int_parser, offsetof(struct config, enable_chunking), "0", sizeof(int)},
|
||||
{ "enable_cjk", "integer", (void*) int_parser, offsetof(struct config, enable_cjk), "0", sizeof(int)},
|
||||
{ "enable_folders", "integer", (void*) int_parser, offsetof(struct config, enable_folders), "0", sizeof(int)},
|
||||
{ "encrypt_messages", "integer", (void*) int_parser, offsetof(struct config, encrypt_messages), "1", sizeof(int)},
|
||||
{ "extra_to_field", "string", (void*) string_parser, offsetof(struct config, extra_to_field), "", MAXVAL-1},
|
||||
{ "extract_attachments", "integer", (void*) int_parser, offsetof(struct config, extract_attachments), "1", sizeof(int)},
|
||||
{ "helper_timeout", "integer", (void*) int_parser, offsetof(struct config, helper_timeout), "20", sizeof(int)},
|
||||
{ "hostid", "string", (void*) string_parser, offsetof(struct config, hostid), HOSTID, MAXVAL-1},
|
||||
{ "iv", "string", (void*) string_parser, offsetof(struct config, iv), "", MAXVAL-1},
|
||||
{ "listen_addr", "string", (void*) string_parser, offsetof(struct config, listen_addr), "0.0.0.0", MAXVAL-1},
|
||||
{ "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
|
||||
{ "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
|
||||
{ "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
|
||||
{ "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
|
||||
{ "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
|
||||
{ "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
|
||||
{ "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
|
||||
{ "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
|
||||
{ "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
|
||||
{ "min_message_size", "integer", (void*) int_parser, offsetof(struct config, min_message_size), "100", sizeof(int)},
|
||||
{ "min_word_len", "integer", (void*) int_parser, offsetof(struct config, min_word_len), "1", sizeof(int)},
|
||||
{ "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct config, mmap_dedup_test), "0", sizeof(int)},
|
||||
{ "mysqlcharset", "string", (void*) string_parser, offsetof(struct config, mysqlcharset), "utf8mb4", MAXVAL-1},
|
||||
{ "mysqlhost", "string", (void*) string_parser, offsetof(struct config, mysqlhost), "", MAXVAL-1},
|
||||
{ "mysqlport", "integer", (void*) int_parser, offsetof(struct config, mysqlport), "", sizeof(int)},
|
||||
{ "mysqlsocket", "string", (void*) string_parser, offsetof(struct config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},
|
||||
{ "mysqluser", "string", (void*) string_parser, offsetof(struct config, mysqluser), "piler", MAXVAL-1},
|
||||
{ "mysqlpwd", "string", (void*) string_parser, offsetof(struct config, mysqlpwd), "", MAXVAL-1},
|
||||
{ "mysqldb", "string", (void*) string_parser, offsetof(struct config, mysqldb), "piler", MAXVAL-1},
|
||||
{ "mysql_connect_timeout", "integer", (void*) int_parser, offsetof(struct config, mysql_connect_timeout), "2", sizeof(int)},
|
||||
{ "number_of_worker_processes", "integer", (void*) int_parser, offsetof(struct config, number_of_worker_processes), "2", sizeof(int)},
|
||||
{ "pemfile", "string", (void*) string_parser, offsetof(struct config, pemfile), "", MAXVAL-1},
|
||||
{ "pidfile", "string", (void*) string_parser, offsetof(struct config, pidfile), PIDFILE, MAXVAL-1},
|
||||
{ "piler_header_field", "string", (void*) string_parser, offsetof(struct config, piler_header_field), "X-piler-id:", MAXVAL-1},
|
||||
{ "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct config, process_rcpt_to_addresses), "0", sizeof(int)},
|
||||
{ "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1},
|
||||
{ "rtindex", "integer", (void*) int_parser, offsetof(struct config, rtindex), "0", sizeof(int)},
|
||||
{ "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1},
|
||||
{ "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)},
|
||||
{ "sphxdb", "string", (void*) string_parser, offsetof(struct config, sphxdb), "piler1", MAXVAL-1},
|
||||
{ "sphxhost", "string", (void*) string_parser, offsetof(struct config, sphxhost), "127.0.0.1", MAXVAL-1},
|
||||
{ "sphxport", "integer", (void*) int_parser, offsetof(struct config, sphxport), "9306", sizeof(int)},
|
||||
{ "smtp_access_list", "integer", (void*) int_parser, offsetof(struct config, smtp_access_list), "0", sizeof(int)},
|
||||
{ "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)},
|
||||
{ "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1},
|
||||
{ "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)},
|
||||
{ "tls_enable", "integer", (void*) int_parser, offsetof(struct config, tls_enable), "0", sizeof(int)},
|
||||
{ "tls_min_version", "string", (void*) string_parser, offsetof(struct config, tls_min_version), "TLSv1.2", MAXVAL-1},
|
||||
{ "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct config, tweak_sent_time_offset), "0", sizeof(int)},
|
||||
{ "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)},
|
||||
{ "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1},
|
||||
{ "use_antivirus", "integer", (void*) int_parser, offsetof(struct config, use_antivirus), "1", sizeof(int)},
|
||||
{ "verbosity", "integer", (void*) int_parser, offsetof(struct config, verbosity), "1", sizeof(int)},
|
||||
{ "workdir", "string", (void*) string_parser, offsetof(struct config, workdir), WORK_DIR, MAXVAL-1},
|
||||
|
||||
{NULL, NULL, NULL, 0, 0, 0}
|
||||
{ "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct __config, archive_emails_not_having_message_id), "0", sizeof(int)},
|
||||
{ "backlog", "integer", (void*) int_parser, offsetof(struct __config, backlog), "20", sizeof(int)},
|
||||
{ "cipher_list", "string", (void*) string_parser, offsetof(struct __config, cipher_list), "HIGH:MEDIUM", MAXVAL-1},
|
||||
{ "clamd_addr", "string", (void*) string_parser, offsetof(struct __config, clamd_addr), "", MAXVAL-1},
|
||||
{ "clamd_port", "integer", (void*) int_parser, offsetof(struct __config, clamd_port), "0", sizeof(int)},
|
||||
{ "clamd_socket", "string", (void*) string_parser, offsetof(struct __config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
|
||||
{ "debug", "integer", (void*) int_parser, offsetof(struct __config, debug), "0", sizeof(int)},
|
||||
{ "default_retention_days", "integer", (void*) int_parser, offsetof(struct __config, default_retention_days), "2557", sizeof(int)},
|
||||
{ "encrypt_messages", "integer", (void*) int_parser, offsetof(struct __config, encrypt_messages), "1", sizeof(int)},
|
||||
{ "extra_to_field", "string", (void*) string_parser, offsetof(struct __config, extra_to_field), "", MAXVAL-1},
|
||||
{ "hostid", "string", (void*) string_parser, offsetof(struct __config, hostid), HOSTID, MAXVAL-1},
|
||||
{ "iv", "string", (void*) string_parser, offsetof(struct __config, iv), "", MAXVAL-1},
|
||||
{ "listen_addr", "string", (void*) string_parser, offsetof(struct __config, listen_addr), "127.0.0.1", MAXVAL-1},
|
||||
{ "listen_port", "integer", (void*) int_parser, offsetof(struct __config, listen_port), "10025", sizeof(int)},
|
||||
{ "locale", "string", (void*) string_parser, offsetof(struct __config, locale), "", MAXVAL-1},
|
||||
{ "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct __config, max_requests_per_child), "1000", sizeof(int)},
|
||||
{ "memcached_servers", "string", (void*) string_parser, offsetof(struct __config, memcached_servers), "127.0.0.1", MAXVAL-1},
|
||||
{ "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct __config, memcached_to_db_interval), "900", sizeof(int)},
|
||||
{ "memcached_ttl", "integer", (void*) int_parser, offsetof(struct __config, memcached_ttl), "86400", sizeof(int)},
|
||||
{ "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)},
|
||||
{ "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1},
|
||||
{ "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)},
|
||||
{ "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},
|
||||
{ "mysqluser", "string", (void*) string_parser, offsetof(struct __config, mysqluser), "piler", MAXVAL-1},
|
||||
{ "mysqlpwd", "string", (void*) string_parser, offsetof(struct __config, mysqlpwd), "", MAXVAL-1},
|
||||
{ "mysqldb", "string", (void*) string_parser, offsetof(struct __config, mysqldb), "piler", MAXVAL-1},
|
||||
{ "mysql_connect_timeout", "integer", (void*) int_parser, offsetof(struct __config, mysql_connect_timeout), "2", sizeof(int)},
|
||||
{ "number_of_worker_processes", "integer", (void*) int_parser, offsetof(struct __config, number_of_worker_processes), "10", sizeof(int)},
|
||||
{ "pemfile", "string", (void*) string_parser, offsetof(struct __config, pemfile), "", MAXVAL-1},
|
||||
{ "pidfile", "string", (void*) string_parser, offsetof(struct __config, pidfile), PIDFILE, MAXVAL-1},
|
||||
{ "piler_header_field", "string", (void*) string_parser, offsetof(struct __config, piler_header_field), "", MAXVAL-1},
|
||||
{ "queuedir", "string", (void*) string_parser, offsetof(struct __config, queuedir), QUEUE_DIR, MAXVAL-1},
|
||||
{ "server_id", "integer", (void*) int_parser, offsetof(struct __config, server_id), "0", sizeof(int)},
|
||||
{ "session_timeout", "integer", (void*) int_parser, offsetof(struct __config, session_timeout), "420", sizeof(int)},
|
||||
{ "spam_header_line", "string", (void*) string_parser, offsetof(struct __config, spam_header_line), "", MAXVAL-1},
|
||||
{ "tls_enable", "integer", (void*) int_parser, offsetof(struct __config, tls_enable), "0", sizeof(int)},
|
||||
{ "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct __config, tweak_sent_time_offset), "0", sizeof(int)},
|
||||
{ "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct __config, update_counters_to_memcached), "0", sizeof(int)},
|
||||
{ "username", "string", (void*) string_parser, offsetof(struct __config, username), "piler", MAXVAL-1},
|
||||
{ "use_antivirus", "integer", (void*) int_parser, offsetof(struct __config, use_antivirus), "1", sizeof(int)},
|
||||
{ "verbosity", "integer", (void*) int_parser, offsetof(struct __config, verbosity), "1", sizeof(int)},
|
||||
{ "workdir", "string", (void*) string_parser, offsetof(struct __config, workdir), WORK_DIR, MAXVAL-1},
|
||||
|
||||
{NULL, NULL, NULL, 0, 0}
|
||||
};
|
||||
|
||||
|
||||
@ -118,7 +110,7 @@ struct _parse_rule config_parse_rules[] =
|
||||
* parse configfile
|
||||
*/
|
||||
|
||||
int parse_config_file(char *configfile, struct config *target_cfg, struct _parse_rule *rules){
|
||||
int parse_config_file(char *configfile, struct __config *target_cfg, struct _parse_rule *rules){
|
||||
char line[MAXVAL], *chpos;
|
||||
FILE *f;
|
||||
|
||||
@ -143,7 +135,7 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse
|
||||
printf("failed to parse %s: \"%s\"\n", line, chpos+1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
@ -158,27 +150,7 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse
|
||||
}
|
||||
|
||||
|
||||
int get_tls_protocol_number(char *protocol){
|
||||
struct tls_protocol tls_protocols[] = {
|
||||
{ "TLSv1", TLS1_VERSION },
|
||||
{ "TLSv1.1", TLS1_1_VERSION },
|
||||
{ "TLSv1.2", TLS1_2_VERSION },
|
||||
#ifdef TLS1_3_VERSION
|
||||
{ "TLSv1.3", TLS1_3_VERSION },
|
||||
#endif
|
||||
};
|
||||
|
||||
for(unsigned int i=0; i<sizeof(tls_protocols)/sizeof(struct tls_protocol); i++){
|
||||
if(!strcmp(protocol, tls_protocols[i].proto)) {
|
||||
return tls_protocols[i].version;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int load_default_config(struct config *cfg, struct _parse_rule *rules){
|
||||
int load_default_config(struct __config *cfg, struct _parse_rule *rules){
|
||||
int i=0;
|
||||
|
||||
while(rules[i].name){
|
||||
@ -194,12 +166,12 @@ int load_default_config(struct config *cfg, struct _parse_rule *rules){
|
||||
* read configuration file variables
|
||||
*/
|
||||
|
||||
struct config read_config(char *configfile){
|
||||
struct config cfg;
|
||||
struct __config read_config(char *configfile){
|
||||
struct __config cfg;
|
||||
|
||||
/* reset config structure and fill it with defaults */
|
||||
|
||||
memset((char *)&cfg, 0, sizeof(struct config));
|
||||
memset((char *)&cfg, 0, sizeof(struct __config));
|
||||
|
||||
load_default_config(&cfg, config_parse_rules);
|
||||
|
||||
@ -208,11 +180,6 @@ struct config read_config(char *configfile){
|
||||
|
||||
if(parse_config_file(configfile, &cfg, config_parse_rules) == -1) printf("error parsing the configfile: %s\n", configfile);
|
||||
|
||||
cfg.hostid_len = strlen(cfg.hostid);
|
||||
|
||||
// Get the TLS protocol constant from string, ie. TLSv1.3 -> 772
|
||||
cfg.tls_min_version_number = get_tls_protocol_number(cfg.tls_min_version);
|
||||
|
||||
return cfg;
|
||||
}
|
||||
|
||||
@ -221,10 +188,9 @@ struct config read_config(char *configfile){
|
||||
* print a single configuration item as key=value
|
||||
*/
|
||||
|
||||
void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
|
||||
void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
|
||||
int j;
|
||||
float f;
|
||||
uint64 u;
|
||||
char *p, buf[MAXVAL];
|
||||
|
||||
p = (char*)cfg + rules[i].offset;
|
||||
@ -233,10 +199,6 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
|
||||
memcpy((char*)&j, p, sizeof(int));
|
||||
printf("%s=%d\n", rules[i].name, j);
|
||||
}
|
||||
else if(strcmp(rules[i].type, "uint64") == 0){
|
||||
memcpy((char*)&u, p, sizeof(uint64));
|
||||
printf("%s=%llu\n", rules[i].name, u);
|
||||
}
|
||||
else if(strcmp(rules[i].type, "float") == 0){
|
||||
memcpy((char*)&f, p, sizeof(float));
|
||||
printf("%s=%.4f\n", rules[i].name, f);
|
||||
@ -253,7 +215,7 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
|
||||
trimBuffer(p);
|
||||
printf("%s=%s\n", rules[i].name, p);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -261,7 +223,7 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
|
||||
* print all known configuration items
|
||||
*/
|
||||
|
||||
void print_config_all(struct config *cfg, char *key){
|
||||
void print_config_all(struct __config *cfg, char *key){
|
||||
int i=0;
|
||||
struct _parse_rule *rules;
|
||||
|
||||
@ -285,7 +247,7 @@ void print_config_all(struct config *cfg, char *key){
|
||||
* print all configuration items found in configfile
|
||||
*/
|
||||
|
||||
void print_config(char *configfile, struct config *cfg){
|
||||
void print_config(char *configfile, struct __config *cfg){
|
||||
FILE *f;
|
||||
char line[MAXVAL], *chpos, previtem[MAXVAL];
|
||||
struct _parse_rule *rules;
|
||||
@ -320,10 +282,12 @@ void print_config(char *configfile, struct config *cfg){
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
|
||||
if(!rules[i].name) printf("unknown key: \"%s\" \n", line);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
|
||||
|
||||
42
src/cfg.h
42
src/cfg.h
@ -7,13 +7,11 @@
|
||||
|
||||
#include "config.h"
|
||||
|
||||
struct config {
|
||||
struct __config {
|
||||
int server_id;
|
||||
char username[MAXVAL];
|
||||
|
||||
char hostid[MAXVAL];
|
||||
int hostid_len;
|
||||
|
||||
char pidfile[MAXVAL];
|
||||
|
||||
char listen_addr[MAXVAL];
|
||||
@ -25,36 +23,27 @@ struct config {
|
||||
|
||||
int encrypt_messages;
|
||||
|
||||
int enable_chunking;
|
||||
int tls_enable;
|
||||
char pemfile[MAXVAL];
|
||||
char cipher_list[MAXVAL];
|
||||
char tls_min_version[MAXVAL];
|
||||
int tls_min_version_number;
|
||||
|
||||
int use_antivirus;
|
||||
|
||||
char memcached_servers[MAXVAL];
|
||||
int memcached_ttl;
|
||||
|
||||
int max_connections;
|
||||
int number_of_worker_processes;
|
||||
int max_requests_per_child;
|
||||
|
||||
int backlog;
|
||||
|
||||
int process_rcpt_to_addresses;
|
||||
|
||||
char workdir[MAXVAL];
|
||||
char queuedir[MAXVAL];
|
||||
|
||||
int verbosity;
|
||||
char locale[MAXVAL];
|
||||
|
||||
int check_for_client_timeout_interval;
|
||||
int smtp_timeout;
|
||||
int helper_timeout;
|
||||
int extract_attachments;
|
||||
int session_timeout;
|
||||
|
||||
char piler_header_field[MAXVAL];
|
||||
char extra_to_field[MAXVAL];
|
||||
@ -66,12 +55,8 @@ struct config {
|
||||
|
||||
int default_retention_days;
|
||||
|
||||
char security_header[MAXVAL];
|
||||
char archive_address[MAXVAL];
|
||||
|
||||
// mysql stuff
|
||||
|
||||
char mysqlcharset[MAXVAL];
|
||||
char mysqlhost[MAXVAL];
|
||||
int mysqlport;
|
||||
char mysqlsocket[MAXVAL];
|
||||
@ -80,39 +65,16 @@ struct config {
|
||||
char mysqldb[MAXVAL];
|
||||
int mysql_connect_timeout;
|
||||
|
||||
// manticore stuff
|
||||
char sphxhost[MAXVAL];
|
||||
int sphxport;
|
||||
char sphxdb[MAXVAL];
|
||||
int rtindex;
|
||||
|
||||
int update_counters_to_memcached;
|
||||
int memcached_to_db_interval;
|
||||
|
||||
int archive_emails_not_having_message_id;
|
||||
|
||||
int archive_only_mydomains;
|
||||
|
||||
int min_word_len;
|
||||
|
||||
int min_message_size;
|
||||
|
||||
int tweak_sent_time_offset;
|
||||
|
||||
int enable_cjk;
|
||||
|
||||
int syslog_recipients;
|
||||
|
||||
int mmap_dedup_test;
|
||||
|
||||
int enable_folders;
|
||||
|
||||
int debug;
|
||||
|
||||
int smtp_access_list;
|
||||
|
||||
int max_message_size;
|
||||
uint64 max_smtp_memory;
|
||||
};
|
||||
|
||||
|
||||
|
||||
87
src/clamd.c
87
src/clamd.c
@ -18,18 +18,20 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int clamd_scan(char *tmpfile, struct config *cfg){
|
||||
int clamd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg){
|
||||
int s, n;
|
||||
char buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
|
||||
char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
|
||||
struct sockaddr_un server;
|
||||
|
||||
memset(avinfo, 0, SMALLBUFSIZE);
|
||||
|
||||
chmod(tmpfile, 0644);
|
||||
|
||||
strcpy(server.sun_path, cfg->clamd_socket);
|
||||
server.sun_family = AF_UNIX;
|
||||
|
||||
if((s = socket(AF_UNIX, SOCK_STREAM, 0)) == -1){
|
||||
syslog(LOG_PRIORITY, "ERR: create socket to %s", cfg->clamd_socket);
|
||||
syslog(LOG_PRIORITY, "ERR: create socket");
|
||||
return AV_ERROR;
|
||||
}
|
||||
|
||||
@ -41,7 +43,7 @@ int clamd_scan(char *tmpfile, struct config *cfg){
|
||||
|
||||
/* issue the SCAN command with full path to the temporary directory */
|
||||
|
||||
|
||||
|
||||
memset(scan_cmd, 0, SMALLBUFSIZE);
|
||||
snprintf(scan_cmd, SMALLBUFSIZE-1, "SCAN %s/%s\r\n", cfg->workdir, tmpfile);
|
||||
|
||||
@ -58,13 +60,13 @@ int clamd_scan(char *tmpfile, struct config *cfg){
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf);
|
||||
|
||||
if(strcasestr(buf, CLAMD_RESP_INFECTED)){
|
||||
char *p = strchr(buf, ' ');
|
||||
p = strchr(buf, ' ');
|
||||
if(p){
|
||||
char *q = strrchr(p, ' ');
|
||||
q = strrchr(p, ' ');
|
||||
if(q){
|
||||
*q = '\0';
|
||||
p++;
|
||||
syslog(LOG_PRIORITY, "%s: VIRUS <%s> found, status=%s", tmpfile, p, S_STATUS_DISCARDED);
|
||||
strncpy(avinfo, p, SMALLBUFSIZE-1);
|
||||
}
|
||||
}
|
||||
|
||||
@ -73,3 +75,74 @@ int clamd_scan(char *tmpfile, struct config *cfg){
|
||||
|
||||
return AV_OK;
|
||||
}
|
||||
|
||||
|
||||
int clamd_net_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg){
|
||||
int n, psd, rc, ret=AV_OK;
|
||||
char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
|
||||
char port_string[6];
|
||||
struct addrinfo hints, *res;
|
||||
|
||||
memset(avinfo, 0, SMALLBUFSIZE);
|
||||
|
||||
chmod(tmpfile, 0644);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to pass to clamd", tmpfile);
|
||||
|
||||
snprintf(port_string, sizeof(port_string)-1, "%d", cfg->clamd_port);
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
if((rc = getaddrinfo(cfg->clamd_addr, port_string, &hints, &res)) != 0){
|
||||
syslog(LOG_PRIORITY, "%s: getaddrinfo for '%s': %s\n", tmpfile, cfg->clamd_addr, gai_strerror(rc));
|
||||
return AV_ERROR;
|
||||
}
|
||||
|
||||
if((psd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
|
||||
syslog(LOG_PRIORITY, "%s: ERR: create socket", tmpfile);
|
||||
ret = AV_ERROR;
|
||||
goto ENDE_CLAMD;
|
||||
}
|
||||
|
||||
if(connect(psd, res->ai_addr, res->ai_addrlen) == -1){
|
||||
syslog(LOG_PRIORITY, "%s: CLAMD ERR: connect to %s %d", tmpfile, cfg->clamd_addr, cfg->clamd_port);
|
||||
ret = AV_ERROR;
|
||||
goto ENDE_CLAMD;
|
||||
}
|
||||
|
||||
|
||||
memset(scan_cmd, 0, SMALLBUFSIZE);
|
||||
snprintf(scan_cmd, SMALLBUFSIZE-1, "SCAN %s/%s\r\n", cfg->workdir, tmpfile);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD CMD: %s", tmpfile, scan_cmd);
|
||||
|
||||
send(psd, scan_cmd, strlen(scan_cmd), 0);
|
||||
|
||||
n = recvtimeout(psd, buf, MAXBUFSIZE, TIMEOUT);
|
||||
close(psd);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf);
|
||||
|
||||
if(strcasestr(buf, CLAMD_RESP_INFECTED)){
|
||||
p = strchr(buf, ' ');
|
||||
if(p){
|
||||
q = strrchr(p, ' ');
|
||||
if(q){
|
||||
*q = '\0';
|
||||
p++;
|
||||
strncpy(avinfo, p, SMALLBUFSIZE-1);
|
||||
}
|
||||
}
|
||||
|
||||
ret = AV_VIRUS;
|
||||
}
|
||||
|
||||
|
||||
ENDE_CLAMD:
|
||||
freeaddrinfo(res);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
58
src/config.h
58
src/config.h
@ -9,21 +9,23 @@
|
||||
#include "piler-config.h"
|
||||
#include "params.h"
|
||||
|
||||
typedef unsigned long long uint64;
|
||||
#define PROGNAME "piler"
|
||||
#define PILERGETD_PROGNAME "pilergetd"
|
||||
|
||||
#define BUILD 1001
|
||||
#define VERSION "0.1.23"
|
||||
|
||||
#define BUILD 789
|
||||
|
||||
#define HOSTID "mailarchiver"
|
||||
|
||||
#define CONFIG_FILE CONFDIR "/piler/piler.conf"
|
||||
#define SMTP_ACL_FILE CONFDIR "/piler/smtp.acl"
|
||||
#define CONFIG_FILE CONFDIR "/piler.conf"
|
||||
#define WORK_DIR DATADIR "/piler/tmp"
|
||||
#define QUEUE_DIR DATADIR "/piler/store"
|
||||
#define ERROR_DIR DATADIR "/piler/error"
|
||||
|
||||
#define CLAMD_SOCKET "/tmp/clamd"
|
||||
|
||||
#define PIDFILE "/var/run/piler/piler.pid"
|
||||
#define PILERGETD_PIDFILE "/var/run/piler/pilergetd.pid"
|
||||
#define QUARANTINELEN 255
|
||||
#define TIMEOUT 60
|
||||
#define TIMEOUT_USEC 500000
|
||||
@ -32,30 +34,13 @@ typedef unsigned long long uint64;
|
||||
#define SMALLBUFSIZE 512
|
||||
#define BIGBUFSIZE 131072
|
||||
#define REALLYBIGBUFSIZE 524288
|
||||
#define SMTPBUFSIZE 2048000
|
||||
#define TINYBUFSIZE 128
|
||||
#define MAXVAL 256
|
||||
#define RANDOM_POOL "/dev/urandom"
|
||||
#define RND_STR_LEN 36
|
||||
#define BUFLEN 32
|
||||
#define QUEUE_ID_LEN 16
|
||||
#define IPLEN 16+1
|
||||
#define KEYLEN 56
|
||||
#define MIN_EMAIL_ADDRESS_LEN 9
|
||||
// Sphinx 3.x has an issue with tokens longer than 41 characters.
|
||||
//
|
||||
// When a regular user executes a query, then his default email address filter
|
||||
// causes the query to fail with the below error message, even when the query
|
||||
// itself is correct:
|
||||
//
|
||||
// SELECT id FROM main1,dailydelta1,delta1 WHERE MATCH(' ( (@sender thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom ) | (@rcpt thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom) ) ') ORDER BY `sent` DESC LIMIT 0,20 OPTION max_matches=1000'
|
||||
//
|
||||
// ERROR 1064 (42000): index dailydelta1,delta1,main1: syntax error, unexpected $end near ' '
|
||||
//
|
||||
// Note that we use 42, because the parser adds a trailing space to the tokens
|
||||
// See https://www.mailpiler.org/wiki/current:sphinx3 and
|
||||
// https://bitbucket.org/jsuto/piler/issues/1082/no-sphinx-results-with-long-email for more
|
||||
#define MAX_EMAIL_ADDRESS_SPHINX_LEN 42
|
||||
|
||||
#define CRLF "\n"
|
||||
|
||||
@ -72,16 +57,12 @@ typedef unsigned long long uint64;
|
||||
#define MEMCACHED_MSGS_DUPLICATE MEMCACHED_CLAPF_PREFIX "duplicate"
|
||||
#define MEMCACHED_MSGS_IGNORE MEMCACHED_CLAPF_PREFIX "ignore"
|
||||
#define MEMCACHED_MSGS_SIZE MEMCACHED_CLAPF_PREFIX "size"
|
||||
#define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size"
|
||||
|
||||
|
||||
#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n"
|
||||
|
||||
#define LOG_PRIORITY LOG_INFO
|
||||
|
||||
#define _LOG_INFO 3
|
||||
#define _LOG_DEBUG 5
|
||||
#define _LOG_EXTREME 100
|
||||
|
||||
#define MAX_RCPT_TO 128
|
||||
|
||||
@ -102,42 +83,30 @@ typedef unsigned long long uint64;
|
||||
#define SQL_RECIPIENT_TABLE "rcpt"
|
||||
#define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
|
||||
#define SQL_RETENTION_RULE_TABLE "retention_rule"
|
||||
#define SQL_FOLDER_RULE_TABLE "folder_rule"
|
||||
#define SQL_COUNTER_TABLE "counter"
|
||||
#define SQL_OPTION_TABLE "option"
|
||||
#define SQL_DOMAIN_TABLE "domain"
|
||||
#define SQL_CUSTOMER_TABLE "customer"
|
||||
#define SQL_IMPORT_TABLE "import"
|
||||
#define SQL_LEGAL_HOLD_TABLE "legal_hold"
|
||||
#define SQL_FOLDER_MESSAGE_TABLE "folder_message"
|
||||
#define SQL_MESSAGES_VIEW "v_messages"
|
||||
#define SQL_ATTACHMENTS_VIEW "v_attachment"
|
||||
|
||||
#define SQL_PREPARED_STMT_GET_DOMAINS "SELECT `domain` FROM `" SQL_DOMAIN_TABLE "`"
|
||||
#define SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID "SELECT id, piler_id FROM " SQL_METADATA_TABLE " WHERE message_id=?"
|
||||
#define SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID "SELECT id FROM " SQL_METADATA_TABLE " WHERE message_id=?"
|
||||
#define SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE "INSERT INTO " SQL_RECIPIENT_TABLE " (`id`,`to`,`todomain`) VALUES(?,?,?)"
|
||||
#define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
#define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
#define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)"
|
||||
#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?"
|
||||
#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=?"
|
||||
#define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?"
|
||||
#define SQL_PREPARED_STMT_QUERY_ATTACHMENT "SELECT `attachment_id`, `ptr` FROM " SQL_ATTACHMENT_TABLE " WHERE piler_id=? ORDER BY attachment_id ASC"
|
||||
#define SQL_PREPARED_STMT_GET_FOLDER_ID "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
|
||||
#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
|
||||
#define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
|
||||
#define SQL_PREPARED_STMT_GET_METADATA_REFERENCE "SELECT COUNT(*) AS count FROM " SQL_METADATA_TABLE " WHERE reference=?"
|
||||
#define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
|
||||
#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
|
||||
#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE "UPDATE " SQL_IMPORT_TABLE " SET started=?, status=?, imported=? WHERE id=?"
|
||||
|
||||
|
||||
/* Error codes */
|
||||
|
||||
#define OK 0
|
||||
#define ERR 1
|
||||
#define ERR_EXISTS 2
|
||||
#define ERR_DISCARDED 3
|
||||
#define ERR_MYDOMAINS 4
|
||||
#define ERR_FOLDER -1
|
||||
|
||||
#define AVIR_OK 0
|
||||
#define AVIR_VIRUS 1
|
||||
@ -151,10 +120,5 @@ typedef unsigned long long uint64;
|
||||
#define WRITE_TO_STDOUT 0
|
||||
#define WRITE_TO_BUFFER 1
|
||||
|
||||
#define S_STATUS_UNDEF "undef"
|
||||
#define S_STATUS_STORED "stored"
|
||||
#define S_STATUS_DUPLICATE "duplicate"
|
||||
#define S_STATUS_DISCARDED "discarded"
|
||||
#define S_STATUS_ERROR "error"
|
||||
|
||||
#endif /* _CONFIG_H */
|
||||
|
||||
|
||||
115
src/counters.c
115
src/counters.c
@ -11,120 +11,119 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
struct counters load_counters(struct session_data *sdata){
|
||||
struct __counters loadCounters(struct session_data *sdata, struct __config *cfg){
|
||||
char buf[SMALLBUFSIZE];
|
||||
struct counters counters;
|
||||
struct sql sql;
|
||||
struct __counters counters;
|
||||
|
||||
bzero(&counters, sizeof(counters));
|
||||
|
||||
snprintf(buf, SMALLBUFSIZE-1, "SELECT `rcvd`, `virus`, `duplicate`, `ignore`, `size`, `stored_size` FROM `%s`", SQL_COUNTER_TABLE);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "SELECT `rcvd`, `virus`, `duplicate`, `ignore`, `size` FROM `%s`", SQL_COUNTER_TABLE);
|
||||
|
||||
#ifdef NEED_MYSQL
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, buf) == ERR) return counters;
|
||||
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK){
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = (char *)&counters.c_rcvd; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&counters.c_virus; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&counters.c_duplicate; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&counters.c_ignore; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&counters.c_size; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&counters.c_stored_size; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
|
||||
p_store_results(&sql);
|
||||
p_fetch_results(&sql);
|
||||
p_free_results(&sql);
|
||||
if(mysql_real_query(&(sdata->mysql), buf, strlen(buf)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res != NULL){
|
||||
row = mysql_fetch_row(res);
|
||||
if(row){
|
||||
counters.c_rcvd = strtoull(row[0], NULL, 10);
|
||||
counters.c_virus = strtoull(row[1], NULL, 10);
|
||||
counters.c_duplicate = strtoull(row[2], NULL, 10);
|
||||
counters.c_ignore = strtoull(row[3], NULL, 10);
|
||||
counters.c_size = strtoull(row[4], NULL, 10);
|
||||
}
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
#endif
|
||||
|
||||
return counters;
|
||||
}
|
||||
|
||||
|
||||
void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
void update_counters(struct session_data *sdata, struct __data *data, struct __counters *counters, struct __config *cfg){
|
||||
char buf[MAXBUFSIZE];
|
||||
#ifdef HAVE_MEMCACHED
|
||||
unsigned long long mc;
|
||||
struct counters c;
|
||||
unsigned long long mc, rcvd;
|
||||
struct __counters c;
|
||||
char key[MAX_MEMCACHED_KEY_LEN];
|
||||
unsigned int flags=0;
|
||||
#endif
|
||||
|
||||
if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size == 0) return;
|
||||
|
||||
#ifdef HAVE_MEMCACHED
|
||||
if(cfg->update_counters_to_memcached == 1){
|
||||
|
||||
/* increment counters to memcached */
|
||||
|
||||
if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
|
||||
unsigned long long rcvd = mc;
|
||||
if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
|
||||
rcvd = mc;
|
||||
|
||||
if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, counters->c_virus, &mc);
|
||||
if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, counters->c_duplicate, &mc);
|
||||
if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, counters->c_ignore, &mc);
|
||||
if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, counters->c_size, &mc);
|
||||
if(counters->c_stored_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, counters->c_stored_size, &mc);
|
||||
if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), counters->c_virus, &mc);
|
||||
if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), counters->c_duplicate, &mc);
|
||||
if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), counters->c_ignore, &mc);
|
||||
if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), counters->c_size, &mc);
|
||||
|
||||
|
||||
bzero(&c, sizeof(c));
|
||||
bzero(&c, sizeof(c));
|
||||
|
||||
snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_MSGS_STORED_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE);
|
||||
snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE);
|
||||
|
||||
if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){
|
||||
char key[MAX_MEMCACHED_KEY_LEN];
|
||||
|
||||
while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){
|
||||
if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10);
|
||||
else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10);
|
||||
else if(!strcmp(key, MEMCACHED_MSGS_DUPLICATE)) c.c_duplicate = strtoull(buf, NULL, 10);
|
||||
else if(!strcmp(key, MEMCACHED_MSGS_IGNORE)) c.c_ignore = strtoull(buf, NULL, 10);
|
||||
else if(!strcmp(key, MEMCACHED_MSGS_SIZE)) c.c_size = strtoull(buf, NULL, 10);
|
||||
else if(!strcmp(key, MEMCACHED_MSGS_STORED_SIZE)) c.c_stored_size = strtoull(buf, NULL, 10);
|
||||
else if(!strcmp(key, MEMCACHED_COUNTERS_LAST_UPDATE)) mc = strtoull(buf, NULL, 10);
|
||||
}
|
||||
|
||||
|
||||
if(sdata->now - mc > (unsigned long long)cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "set", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
|
||||
|
||||
snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=%llu, `virus`=%llu, `duplicate`=%llu, `ignore`=%llu, `size`=%llu, `stored_size`=%llu", SQL_COUNTER_TABLE, c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore, c.c_size, c.c_stored_size);
|
||||
if(sdata->now - mc > cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_set(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
|
||||
|
||||
snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=%llu, `virus`=%llu, `duplicate`=%llu, `ignore`=%llu, `size`=%llu", SQL_COUNTER_TABLE, c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore, c.c_size);
|
||||
|
||||
//if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);
|
||||
|
||||
p_query(sdata, buf);
|
||||
goto EXEC_SQL;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
|
||||
c = load_counters(sdata);
|
||||
c = loadCounters(sdata, cfg);
|
||||
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "add", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
|
||||
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_VIRUS, buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_RCVD, buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_DUPLICATE, buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_IGNORE, buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_SIZE, buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_stored_size + counters->c_stored_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_STORED_SIZE, buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), buf, strlen(buf), 0, 0);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), buf, strlen(buf), 0, 0);
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
#endif
|
||||
snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=`rcvd`+%llu, `virus`=`virus`+%llu, `duplicate`=`duplicate`+%llu, `ignore`=`ignore`+%llu, `size`=`size`+%llu, `stored_size`=`stored_size`+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore, counters->c_size, counters->c_stored_size);
|
||||
p_query(sdata, buf);
|
||||
snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=`rcvd`+%llu, `virus`=`virus`+%llu, `duplicate`=`duplicate`+%llu, `ignore`=`ignore`+%llu, `size`=`size`+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore, counters->c_size);
|
||||
|
||||
//if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);
|
||||
|
||||
#ifdef HAVE_MEMCACHED
|
||||
EXEC_SQL:
|
||||
#endif
|
||||
|
||||
#ifdef NEED_MYSQL
|
||||
mysql_real_query(&(sdata->mysql), buf, strlen(buf));
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_MEMCACHED
|
||||
}
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
195
src/decoder.c
195
src/decoder.c
@ -6,7 +6,6 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <ctype.h>
|
||||
#include <iconv.h>
|
||||
#include "decoder.h"
|
||||
#include "htmlentities.h"
|
||||
#include "config.h"
|
||||
@ -67,38 +66,21 @@ static int compmi(const void *m1, const void *m2){
|
||||
}
|
||||
|
||||
|
||||
inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, int *len){
|
||||
int count=0;
|
||||
void sanitiseBase64(char *s){
|
||||
char *p1;
|
||||
|
||||
memset(buf, 0, buflen);
|
||||
if(s == NULL) return;
|
||||
|
||||
/*
|
||||
* Code point 1st byte 2nd byte 3rd byte 4th byte
|
||||
* ---------- -------- -------- -------- --------
|
||||
* U+0000..U+007F 00..7F
|
||||
* U+0080..U+07FF C2..DF 80..BF
|
||||
* U+0800..U+0FFF E0 A0..BF 80..BF
|
||||
*
|
||||
* FIXME: See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#G7404 for valid sequences
|
||||
*/
|
||||
|
||||
if(c <= 0x7F){
|
||||
*(buf+count) = c;
|
||||
count++;
|
||||
for(; *s; s++){
|
||||
if(b64[(unsigned int)(*s & 0xFF)] == 255){
|
||||
for(p1 = s; p1[0] != '\0'; p1++)
|
||||
p1[0] = p1[1];
|
||||
}
|
||||
|
||||
else {
|
||||
*(buf+count) = ( 0xC0 | (c >> 6) );
|
||||
count++;
|
||||
*(buf+count) = ( 0x80 | (c & 0x3F) );
|
||||
count++;
|
||||
}
|
||||
|
||||
*len = count;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
inline static void pack_4_into_3(char *s, char *s2){
|
||||
inline void pack_4_into_3(char *s, char *s2){
|
||||
int j, n[4], k1, k2;
|
||||
|
||||
memset(s2, 0, 3);
|
||||
@ -129,12 +111,32 @@ inline static void pack_4_into_3(char *s, char *s2){
|
||||
|
||||
|
||||
int decodeBase64(char *p){
|
||||
int len=0;
|
||||
unsigned char puf[MAXBUFSIZE];
|
||||
int i, len=0;
|
||||
char s[5], s2[3], puf[MAXBUFSIZE];
|
||||
|
||||
memset(puf, 0, sizeof(puf));
|
||||
if(strlen(p) < 4 || strlen(p) > MAXBUFSIZE/2)
|
||||
return 0;
|
||||
|
||||
len = decode_base64_to_buffer(p, strlen(p), &puf[0], sizeof(puf)-1);
|
||||
for(i=0; i<strlen(p); i++){
|
||||
memcpy(s, p+i, 4);
|
||||
s[4] = '\0';
|
||||
|
||||
i += 3;
|
||||
|
||||
/* safety check against abnormally long lines */
|
||||
|
||||
if(len + 3 > sizeof(puf)-1) break;
|
||||
|
||||
if(strlen(s) == 4){
|
||||
pack_4_into_3(s, s2);
|
||||
memcpy(puf+len, s2, 3);
|
||||
|
||||
len += 3;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
*(puf+len) = '\0';
|
||||
|
||||
snprintf(p, MAXBUFSIZE-1, "%s", puf);
|
||||
|
||||
@ -149,23 +151,21 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
|
||||
if(plen < 4 || plen > blen)
|
||||
return 0;
|
||||
|
||||
for(i=0; i<plen; i+=4){
|
||||
for(i=0; i<plen; i++){
|
||||
memcpy(s, p+i, 4);
|
||||
s[4] = '\0';
|
||||
int decodedlen = 3;
|
||||
|
||||
i += 3;
|
||||
|
||||
/* safety check against abnormally long lines */
|
||||
|
||||
if(len + decodedlen > blen-1) break;
|
||||
if(len + 3 > blen-1) break;
|
||||
|
||||
if(strlen(s) == 4){
|
||||
pack_4_into_3(s, s2);
|
||||
if(s[3] == '=') decodedlen = 2;
|
||||
if(s[2] == '=') decodedlen = 1;
|
||||
memcpy(b+len, s2, 3);
|
||||
|
||||
memcpy(b+len, s2, decodedlen);
|
||||
|
||||
len += decodedlen;
|
||||
len += 3;
|
||||
}
|
||||
|
||||
}
|
||||
@ -175,13 +175,13 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
|
||||
|
||||
|
||||
void decodeQP(char *p){
|
||||
unsigned int i;
|
||||
int k=0, a, b;
|
||||
int i, k=0, a, b;
|
||||
char c;
|
||||
|
||||
if(p == NULL) return;
|
||||
|
||||
for(i=0; i<strlen((char*)p); i++){
|
||||
char c = p[i];
|
||||
c = p[i];
|
||||
|
||||
if(p[i] == '=' && isxdigit(p[i+1]) && isxdigit(p[i+2])){
|
||||
a = p[i+1];
|
||||
@ -191,20 +191,16 @@ void decodeQP(char *p){
|
||||
|
||||
i += 2;
|
||||
}
|
||||
else if(p[i] == '_'){
|
||||
c = ' ';
|
||||
}
|
||||
|
||||
p[k] = c;
|
||||
k++;
|
||||
|
||||
}
|
||||
|
||||
p[k] = '\0';
|
||||
}
|
||||
|
||||
|
||||
void decodeHTML(char *p, int utf8){
|
||||
void decodeHTML(char *p){
|
||||
unsigned char buf[MAXBUFSIZE], __u[8];
|
||||
char *s, *q;
|
||||
int count=0, len, c;
|
||||
@ -234,16 +230,9 @@ void decodeHTML(char *p, int utf8){
|
||||
res = bsearch(&key, htmlentities, NUM_OF_HTML_ENTITIES, sizeof(struct mi), compmi);
|
||||
|
||||
if(res && res->val <= 255){
|
||||
|
||||
if(utf8 == 1){
|
||||
utf8_encode_char(res->val, &__u[0], sizeof(__u), &len);
|
||||
memcpy(&buf[count], &__u[0], len);
|
||||
count += len;
|
||||
}
|
||||
else {
|
||||
buf[count] = res->val;
|
||||
count++;
|
||||
}
|
||||
utf8_encode_char(res->val, &__u[0], sizeof(__u), &len);
|
||||
memcpy(&buf[count], &__u[0], len);
|
||||
count += len;
|
||||
}
|
||||
else {
|
||||
buf[count] = 'q';
|
||||
@ -268,8 +257,7 @@ void decodeHTML(char *p, int utf8){
|
||||
|
||||
|
||||
void decodeURL(char *p){
|
||||
unsigned int i;
|
||||
int c, k=0, a, b;
|
||||
int i, c, k=0, a, b;
|
||||
|
||||
if(p == NULL) return;
|
||||
|
||||
@ -307,37 +295,76 @@ void decodeURL(char *p){
|
||||
}
|
||||
|
||||
|
||||
int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *encoding){
|
||||
iconv_t cd;
|
||||
size_t inbytesleft, outbytesleft;
|
||||
int ret = ERR;
|
||||
inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, int *len){
|
||||
int count=0;
|
||||
|
||||
memset(outbuf, 0, outbuflen);
|
||||
memset(buf, 0, buflen);
|
||||
|
||||
// Iconv sometimes produces an invalid utf8 sequence for gb2312.
|
||||
// The fix is to use cp936, instead of gb2312 encoding.
|
||||
//
|
||||
// If there will be more similar exceptions, then we have to use
|
||||
// a more efficient lookup method
|
||||
/*
|
||||
* Code point 1st byte 2nd byte 3rd byte 4th byte
|
||||
* ---------- -------- -------- -------- --------
|
||||
* U+0000..U+007F 00..7F
|
||||
* U+0080..U+07FF C2..DF 80..BF
|
||||
* U+0800..U+0FFF E0 A0..BF 80..BF
|
||||
*/
|
||||
|
||||
if(strcasecmp(encoding, "gb2312") == 0)
|
||||
cd = iconv_open("utf-8", "cp936");
|
||||
else if(strcasecmp(encoding, "ks_c_5601-1987") == 0)
|
||||
cd = iconv_open("utf-8", "EUC-KR");
|
||||
else
|
||||
cd = iconv_open("utf-8", encoding);
|
||||
if(c <= 0x7F){
|
||||
*(buf+count) = c;
|
||||
count++;
|
||||
}
|
||||
|
||||
if(cd != (iconv_t)-1){
|
||||
inbytesleft = inbuflen;
|
||||
outbytesleft = outbuflen-1;
|
||||
else if(c <= 0x7FF){
|
||||
*(buf+count) = ( 0xC0 | (c >> 6) );
|
||||
count++;
|
||||
*(buf+count) = ( 0x80 | (c & 0x3F) );
|
||||
count++;
|
||||
}
|
||||
|
||||
if(iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft) == (size_t) -1)
|
||||
ret = ERR;
|
||||
else
|
||||
ret = OK;
|
||||
|
||||
iconv_close(cd);
|
||||
else if (c <= 0xFFFF){
|
||||
*(buf+count) = ( 0xE0 | (c >> 12) );
|
||||
count++;
|
||||
*(buf+count) = ( 0x80 | ((c >> 6) & 0x3F) );
|
||||
count++;
|
||||
*(buf+count) = ( 0x80 | (c & 0x3F) );
|
||||
count++;
|
||||
}
|
||||
|
||||
*len = count;
|
||||
}
|
||||
|
||||
|
||||
void utf8_encode(unsigned char *p){
|
||||
int count=0, len;
|
||||
unsigned char *u, *s, utf8[MAXBUFSIZE], __u[8];
|
||||
|
||||
if(p == NULL || strlen((char *)p) == 0) return;
|
||||
|
||||
memset(utf8, 0, MAXBUFSIZE);
|
||||
u = &utf8[0];
|
||||
s = p;
|
||||
|
||||
for(; *s; s++){
|
||||
|
||||
utf8_encode_char(*s, &__u[0], sizeof(__u), &len);
|
||||
|
||||
/*
|
||||
* this condition should never happen, as according to the RFCs:
|
||||
*
|
||||
* "Each line of characters MUST be no more than 998 characters, and
|
||||
* SHOULD be no more than 78 characters, excluding the CRLF."
|
||||
*
|
||||
*/
|
||||
|
||||
if(count+len > sizeof(utf8)-1) break;
|
||||
|
||||
//printf("%s", __u);
|
||||
memcpy(u+count, &__u[0], len);
|
||||
|
||||
count += len;
|
||||
}
|
||||
|
||||
return ret;
|
||||
*(u+count) = '\0'; count++;
|
||||
memcpy(p, u, count);
|
||||
}
|
||||
|
||||
|
||||
@ -5,11 +5,15 @@
|
||||
#ifndef _DECODER_H
|
||||
#define _DECODER_H
|
||||
|
||||
void base64_encode(unsigned char *in, int inlen, char *out, int outlen);
|
||||
|
||||
void sanitiseBase64(char *s);
|
||||
int decodeBase64(char *p);
|
||||
int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen);
|
||||
void decodeQP(char *p);
|
||||
void decodeHTML(char *p, int utf8);
|
||||
void decodeHTML(char *p);
|
||||
void decodeURL(char *p);
|
||||
int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *encoding);
|
||||
inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, int *len);
|
||||
void utf8_encode(unsigned char *p);
|
||||
|
||||
#endif /* _DECODER_H */
|
||||
|
||||
285
src/defs.h
285
src/defs.h
@ -7,17 +7,27 @@
|
||||
|
||||
#ifdef NEED_MYSQL
|
||||
#include <mysql.h>
|
||||
#include <mysqld_error.h>
|
||||
#endif
|
||||
#ifdef NEED_SQLITE3
|
||||
#include <sqlite3.h>
|
||||
|
||||
/* for older versions of sqlite3 do not have the sqlite3_prepare_v2() function, 2009.12.30, SJ */
|
||||
|
||||
#if SQLITE_VERSION_NUMBER < 3006000
|
||||
#define sqlite3_prepare_v2 sqlite3_prepare
|
||||
#endif
|
||||
|
||||
#endif
|
||||
#ifdef HAVE_TRE
|
||||
#include <tre/tre.h>
|
||||
#include <tre/regex.h>
|
||||
#endif
|
||||
#ifdef HAVE_LIBWRAP
|
||||
#include <tcpd.h>
|
||||
#endif
|
||||
|
||||
#include <openssl/md5.h>
|
||||
#include <openssl/sha.h>
|
||||
#include <openssl/ssl.h>
|
||||
#include <netinet/in.h>
|
||||
#include "tai.h"
|
||||
#include "config.h"
|
||||
|
||||
@ -25,47 +35,35 @@
|
||||
#define MSG_BODY 0
|
||||
#define MSG_RECEIVED 1
|
||||
#define MSG_FROM 2
|
||||
#define MSG_SENDER 3
|
||||
#define MSG_TO 4
|
||||
#define MSG_CC 5
|
||||
#define MSG_SUBJECT 6
|
||||
#define MSG_CONTENT_TYPE 7
|
||||
#define MSG_CONTENT_TRANSFER_ENCODING 8
|
||||
#define MSG_CONTENT_DISPOSITION 9
|
||||
#define MSG_MESSAGE_ID 10
|
||||
#define MSG_REFERENCES 11
|
||||
#define MSG_RECIPIENT 12
|
||||
#define MSG_ENVELOPE_TO 13
|
||||
#define MSG_TO 3
|
||||
#define MSG_CC 4
|
||||
#define MSG_SUBJECT 5
|
||||
#define MSG_CONTENT_TYPE 6
|
||||
#define MSG_CONTENT_TRANSFER_ENCODING 7
|
||||
#define MSG_CONTENT_DISPOSITION 8
|
||||
#define MSG_MESSAGE_ID 9
|
||||
#define MSG_REFERENCES 10
|
||||
#define MSG_RECIPIENT 11
|
||||
|
||||
#define MAXHASH 277
|
||||
#define MAXHASH 8171
|
||||
|
||||
#define BASE64_RATIO 1.33333333
|
||||
|
||||
#define DIGEST_LENGTH EVP_MAX_MD_SIZE
|
||||
#define DIGEST_HEX_LENGTH 2*DIGEST_LENGTH+1
|
||||
#define DIGEST_LENGTH SHA256_DIGEST_LENGTH
|
||||
|
||||
#define UNDEF 0
|
||||
#define READY 1
|
||||
#define BUSY 2
|
||||
|
||||
#define MAX_SQL_VARS 20
|
||||
|
||||
#define TYPE_UNDEF 0
|
||||
#define TYPE_SHORT 1
|
||||
#define TYPE_LONG 2
|
||||
#define TYPE_LONGLONG 3
|
||||
#define TYPE_STRING 4
|
||||
|
||||
#define MAXCHILDREN 64
|
||||
|
||||
#define RULE_UNDEF 0
|
||||
#define RULE_MATCH 1
|
||||
#define RULE_NO_MATCH -100
|
||||
|
||||
typedef void signal_func (int);
|
||||
|
||||
|
||||
struct child {
|
||||
pid_t pid;
|
||||
int serial;
|
||||
int messages;
|
||||
int status;
|
||||
};
|
||||
@ -76,7 +74,7 @@ struct attachment {
|
||||
char type[TINYBUFSIZE];
|
||||
char shorttype[TINYBUFSIZE];
|
||||
char aname[TINYBUFSIZE];
|
||||
char filename[SMALLBUFSIZE];
|
||||
char filename[TINYBUFSIZE];
|
||||
char internalname[TINYBUFSIZE];
|
||||
char digest[2*DIGEST_LENGTH+1];
|
||||
char dumped;
|
||||
@ -90,29 +88,9 @@ struct ptr_array {
|
||||
};
|
||||
|
||||
|
||||
struct node {
|
||||
void *str;
|
||||
unsigned int key;
|
||||
struct node *r;
|
||||
};
|
||||
|
||||
|
||||
struct smtp_acl {
|
||||
char network_str[BUFLEN];
|
||||
in_addr_t low, high;
|
||||
int prefix;
|
||||
int rejected;
|
||||
struct smtp_acl *r;
|
||||
};
|
||||
|
||||
|
||||
struct net {
|
||||
int socket;
|
||||
int use_ssl;
|
||||
int starttls;
|
||||
int timeout;
|
||||
SSL_CTX *ctx;
|
||||
SSL *ssl;
|
||||
struct list {
|
||||
char s[SMALLBUFSIZE];
|
||||
struct list *r;
|
||||
};
|
||||
|
||||
|
||||
@ -121,8 +99,6 @@ struct rule {
|
||||
regex_t from;
|
||||
regex_t to;
|
||||
regex_t subject;
|
||||
regex_t body;
|
||||
regex_t attachment_name;
|
||||
regex_t attachment_type;
|
||||
#endif
|
||||
int spam;
|
||||
@ -131,13 +107,7 @@ struct rule {
|
||||
int attachment_size;
|
||||
char _attachment_size[4];
|
||||
|
||||
char *domain;
|
||||
int domainlen;
|
||||
|
||||
int days;
|
||||
int folder_id;
|
||||
|
||||
char emptyfrom, emptyto, emptysubject, emptybody, emptyaname, emptyatype;
|
||||
|
||||
char *rulestr;
|
||||
char compiled;
|
||||
@ -146,21 +116,7 @@ struct rule {
|
||||
};
|
||||
|
||||
|
||||
struct rule_cond {
|
||||
char domain[SMALLBUFSIZE];
|
||||
char from[SMALLBUFSIZE];
|
||||
char to[SMALLBUFSIZE];
|
||||
char subject[SMALLBUFSIZE];
|
||||
char body[SMALLBUFSIZE];
|
||||
char _size[SMALLBUFSIZE];
|
||||
char attachment_name[SMALLBUFSIZE];
|
||||
char attachment_type[SMALLBUFSIZE];
|
||||
char _attachment_size[SMALLBUFSIZE];
|
||||
int size, attachment_size, spam, days, folder_id;
|
||||
};
|
||||
|
||||
|
||||
struct parser_state {
|
||||
struct _state {
|
||||
int line_num;
|
||||
int message_state;
|
||||
int is_header;
|
||||
@ -173,10 +129,8 @@ struct parser_state {
|
||||
int qp;
|
||||
int htmltag;
|
||||
int style;
|
||||
int meta_content_type;
|
||||
int skip_html;
|
||||
int has_to_dump;
|
||||
int has_to_dump_whole_body;
|
||||
int fd;
|
||||
int b64fd;
|
||||
int mfd;
|
||||
@ -185,47 +139,36 @@ struct parser_state {
|
||||
int content_type_is_set;
|
||||
int pushed_pointer;
|
||||
int saved_size;
|
||||
unsigned int writebufpos;
|
||||
unsigned int abufpos;
|
||||
unsigned int received_header;
|
||||
int writebufpos;
|
||||
int abufpos;
|
||||
char attachedfile[RND_STR_LEN+SMALLBUFSIZE];
|
||||
char message_id[SMALLBUFSIZE];
|
||||
char message_id_hash[2*DIGEST_LENGTH+1];
|
||||
char miscbuf[MAX_TOKEN_LEN];
|
||||
char qpbuf[MAX_TOKEN_LEN];
|
||||
char receivedbuf[SMALLBUFSIZE];
|
||||
unsigned long n_token;
|
||||
unsigned long n_subject_token;
|
||||
unsigned long n_body_token;
|
||||
unsigned long n_chain_token;
|
||||
|
||||
char filename[TINYBUFSIZE];
|
||||
char type[TINYBUFSIZE];
|
||||
char charset[TINYBUFSIZE];
|
||||
|
||||
char attachment_name_buf[SMALLBUFSIZE];
|
||||
int anamepos;
|
||||
|
||||
struct node *boundaries[MAXHASH];
|
||||
struct node *rcpt[MAXHASH];
|
||||
struct node *rcpt_domain[MAXHASH];
|
||||
struct node *journal_recipient[MAXHASH];
|
||||
struct list *boundaries;
|
||||
struct list *rcpt;
|
||||
struct list *rcpt_domain;
|
||||
struct list *journal_recipient;
|
||||
|
||||
int n_attachments;
|
||||
struct attachment attachments[MAX_ATTACHMENTS];
|
||||
|
||||
char reference[SMALLBUFSIZE];
|
||||
|
||||
char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_sender[SMALLBUFSIZE], b_sender_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
|
||||
char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
|
||||
char b_journal_to[MAXBUFSIZE];
|
||||
|
||||
unsigned int bodylen;
|
||||
unsigned int tolen;
|
||||
unsigned int todomainlen;
|
||||
unsigned int found_security_header;
|
||||
|
||||
long unsigned int journaltolen;
|
||||
|
||||
int retention;
|
||||
int bodylen;
|
||||
int tolen;
|
||||
int journaltolen;
|
||||
};
|
||||
|
||||
|
||||
@ -237,23 +180,22 @@ struct session_data {
|
||||
char acceptbuf[SMALLBUFSIZE];
|
||||
char attachments[SMALLBUFSIZE];
|
||||
char internal_sender, internal_recipient, external_recipient;
|
||||
uint64 duplicate_id;
|
||||
short int customer_id;
|
||||
int direction;
|
||||
int tls;
|
||||
int spam_message;
|
||||
int fd, hdr_len, tot_len, stored_len, num_of_rcpt_to, rav;
|
||||
int fd, hdr_len, tot_len, num_of_rcpt_to, rav;
|
||||
int need_scan;
|
||||
float __acquire, __parsed, __av, __store, __compress, __encrypt;
|
||||
char bodydigest[2*DIGEST_LENGTH+1];
|
||||
char digest[2*DIGEST_LENGTH+1];
|
||||
time_t now, sent, delivered, retained;
|
||||
char ms_journal;
|
||||
char import;
|
||||
int journal_envelope_length, journal_bottom_length;
|
||||
unsigned int sql_errno;
|
||||
#ifdef NEED_MYSQL
|
||||
MYSQL mysql, sphx;
|
||||
MYSQL mysql;
|
||||
#endif
|
||||
#ifdef NEED_SQLITE3
|
||||
sqlite3 *db;
|
||||
#endif
|
||||
};
|
||||
|
||||
@ -298,134 +240,47 @@ struct memcached_server {
|
||||
#endif
|
||||
|
||||
|
||||
struct import {
|
||||
char *extra_recipient;
|
||||
char *move_folder;
|
||||
char *failed_folder;
|
||||
int status;
|
||||
int total_messages;
|
||||
int processed_messages;
|
||||
int batch_processing_limit;
|
||||
int start_position;
|
||||
int import_job_id;
|
||||
int remove_after_import;
|
||||
int download_only;
|
||||
int keep_eml;
|
||||
int timeout;
|
||||
int cap_uidplus;
|
||||
int fd;
|
||||
long total_size;
|
||||
int dryrun;
|
||||
int tot_msgs;
|
||||
int port;
|
||||
int seq;
|
||||
int table_id;
|
||||
int delay;
|
||||
char *server;
|
||||
char *username;
|
||||
char *password;
|
||||
char *database;
|
||||
char *skiplist;
|
||||
char *folder_imap;
|
||||
char *folder_name;
|
||||
char *mboxdir;
|
||||
char *folder;
|
||||
char filename[SMALLBUFSIZE];
|
||||
time_t started, updated, finished, after, before;
|
||||
};
|
||||
|
||||
|
||||
struct licence {
|
||||
char customer[TINYBUFSIZE];
|
||||
char hostname[TINYBUFSIZE];
|
||||
char ip[TINYBUFSIZE];
|
||||
time_t expiry;
|
||||
int licenced_users;
|
||||
};
|
||||
|
||||
|
||||
struct data {
|
||||
int folder, quiet;
|
||||
struct __data {
|
||||
int folder;
|
||||
char recursive_folder_names;
|
||||
char starttls[TINYBUFSIZE];
|
||||
struct node *mydomains[MAXHASH];
|
||||
struct node *imapfolders[MAXHASH];
|
||||
struct import *import;
|
||||
struct licence licence;
|
||||
char *dedup;
|
||||
int child_serial;
|
||||
int pos;
|
||||
char mydomains[MAXBUFSIZE];
|
||||
|
||||
#ifdef NEED_MYSQL
|
||||
MYSQL_STMT *stmt_get_meta_id_by_message_id;
|
||||
MYSQL_STMT *stmt_insert_into_rcpt_table;
|
||||
MYSQL_STMT *stmt_insert_into_sphinx_table;
|
||||
MYSQL_STMT *stmt_insert_into_meta_table;
|
||||
MYSQL_STMT *stmt_insert_into_attachment_table;
|
||||
MYSQL_STMT *stmt_get_attachment_id_by_signature;
|
||||
MYSQL_STMT *stmt_get_attachment_pointer;
|
||||
MYSQL_STMT *stmt_query_attachment;
|
||||
MYSQL_STMT *stmt_get_folder_id;
|
||||
MYSQL_STMT *stmt_insert_into_folder_table;
|
||||
MYSQL_STMT *stmt_update_metadata_reference;
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_TRE
|
||||
struct node *archiving_rules[1];
|
||||
struct node *retention_rules[1];
|
||||
struct node *folder_rules[1];
|
||||
struct rule *archiving_rules;
|
||||
struct rule *retention_rules;
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_MEMCACHED
|
||||
struct memcached_server memc;
|
||||
#endif
|
||||
|
||||
struct net *net;
|
||||
SSL_CTX *ctx;
|
||||
SSL *ssl;
|
||||
};
|
||||
|
||||
|
||||
#if !defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID) && \
|
||||
MYSQL_VERSION_ID >= 80001 && MYSQL_VERSION_ID != 80002
|
||||
typedef bool my_bool;
|
||||
#endif
|
||||
|
||||
struct sql {
|
||||
#ifdef NEED_MYSQL
|
||||
MYSQL_STMT *stmt;
|
||||
#endif
|
||||
char *sql[MAX_SQL_VARS];
|
||||
int type[MAX_SQL_VARS];
|
||||
int len[MAX_SQL_VARS];
|
||||
unsigned long length[MAX_SQL_VARS];
|
||||
my_bool is_null[MAX_SQL_VARS];
|
||||
my_bool error[MAX_SQL_VARS];
|
||||
int pos;
|
||||
};
|
||||
|
||||
|
||||
struct counters {
|
||||
struct __counters {
|
||||
unsigned long long c_rcvd;
|
||||
unsigned long long c_virus;
|
||||
unsigned long long c_duplicate;
|
||||
unsigned long long c_ignore;
|
||||
unsigned long long c_size;
|
||||
unsigned long long c_stored_size;
|
||||
};
|
||||
|
||||
|
||||
struct smtp_session {
|
||||
char ttmpfile[SMALLBUFSIZE];
|
||||
char mailfrom[SMALLBUFSIZE];
|
||||
char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
|
||||
char remote_host[INET6_ADDRSTRLEN+1];
|
||||
char nullbyte;
|
||||
time_t lasttime;
|
||||
int protocol_state;
|
||||
int slot;
|
||||
int fd;
|
||||
int bad;
|
||||
int tot_len;
|
||||
int bdat_bytes_to_read;
|
||||
int num_of_rcpt_to;
|
||||
struct config *cfg;
|
||||
struct net net;
|
||||
int max_message_size;
|
||||
char *buf;
|
||||
int buflen;
|
||||
int bufsize;
|
||||
int too_big;
|
||||
int mail_size;
|
||||
};
|
||||
|
||||
struct tls_protocol {
|
||||
char *proto;
|
||||
int version;
|
||||
};
|
||||
|
||||
#endif /* _DEFS_H */
|
||||
|
||||
|
||||
126
src/digest.c
126
src/digest.c
@ -19,7 +19,6 @@
|
||||
|
||||
int search_header_end(char *p, int n){
|
||||
int hdr_len=0;
|
||||
char *q = p;
|
||||
|
||||
if(n < 5) return hdr_len;
|
||||
|
||||
@ -29,43 +28,30 @@ int search_header_end(char *p, int n){
|
||||
hdr_len++;
|
||||
}
|
||||
|
||||
if(*(p-1) == '\n' && strcasestr(q, "Message-ID:")) return n; else return 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int make_digests(struct session_data *sdata, struct config *cfg){
|
||||
int n, fd, offset=3, hdr_len=0;
|
||||
int make_digests(struct session_data *sdata, struct __config *cfg){
|
||||
int i=0, n, fd, offset=3, hdr_len=0, len=0;
|
||||
char *body=NULL;
|
||||
unsigned char buf[BIGBUFSIZE];
|
||||
|
||||
EVP_MD_CTX *ctx, *ctx2;
|
||||
const EVP_MD *md, *md2;
|
||||
unsigned int i=0, md_len, md_len2;
|
||||
unsigned char md_value[EVP_MAX_MD_SIZE], md_value2[EVP_MAX_MD_SIZE];
|
||||
unsigned char buf[BIGBUFSIZE], md[DIGEST_LENGTH], md2[DIGEST_LENGTH];
|
||||
SHA256_CTX context, context2;
|
||||
|
||||
memset(sdata->bodydigest, 0, 2*DIGEST_LENGTH+1);
|
||||
memset(sdata->digest, 0, 2*DIGEST_LENGTH+1);
|
||||
SHA256_Init(&context);
|
||||
SHA256_Init(&context2);
|
||||
|
||||
md = EVP_get_digestbyname("sha256");
|
||||
md2 = EVP_get_digestbyname("sha256");
|
||||
if(md == NULL || md2 == NULL){
|
||||
syslog(LOG_PRIORITY, "ERROR: unknown message digest: sha256 in %s:%d", __func__, __LINE__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
ctx = EVP_MD_CTX_new();
|
||||
EVP_DigestInit_ex(ctx, md, NULL);
|
||||
|
||||
ctx2 = EVP_MD_CTX_new();
|
||||
EVP_DigestInit_ex(ctx2, md2, NULL);
|
||||
|
||||
fd = open(sdata->filename, O_RDONLY);
|
||||
if(fd == -1) return -1;
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
|
||||
while((n = read(fd, buf, sizeof(buf))) > 0){
|
||||
EVP_DigestUpdate(ctx2, buf, n);
|
||||
len += n;
|
||||
|
||||
SHA256_Update(&context2, buf, n);
|
||||
|
||||
body = (char *)&buf[0];
|
||||
|
||||
@ -82,7 +68,7 @@ int make_digests(struct session_data *sdata, struct config *cfg){
|
||||
}
|
||||
|
||||
|
||||
EVP_DigestUpdate(ctx, body, n);
|
||||
SHA256_Update(&context, body, n);
|
||||
|
||||
i++;
|
||||
}
|
||||
@ -91,93 +77,59 @@ int make_digests(struct session_data *sdata, struct config *cfg){
|
||||
|
||||
sdata->hdr_len = hdr_len;
|
||||
|
||||
EVP_DigestFinal_ex(ctx, md_value, &md_len);
|
||||
EVP_MD_CTX_free(ctx);
|
||||
EVP_DigestFinal_ex(ctx2, md_value2, &md_len2);
|
||||
EVP_MD_CTX_free(ctx2);
|
||||
SHA256_Final(md, &context);
|
||||
SHA256_Final(md2, &context2);
|
||||
|
||||
for(i=0;i<md_len;i++){
|
||||
snprintf(sdata->bodydigest + i*2, 3, "%02x", md_value[i]);
|
||||
}
|
||||
|
||||
for(i=0;i<md_len2;i++){
|
||||
snprintf(sdata->digest + i*2, 3, "%02x", md_value2[i]);
|
||||
for(i=0;i<DIGEST_LENGTH;i++){
|
||||
snprintf(sdata->bodydigest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
|
||||
snprintf(sdata->digest + i*2, 2*DIGEST_LENGTH, "%02x", md2[i]);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
void raw_digest_file(char *digestname, char *filename, unsigned char *md_value){
|
||||
int fd, n;
|
||||
unsigned char buf[MAXBUFSIZE];
|
||||
EVP_MD_CTX *ctx;
|
||||
const EVP_MD *md;
|
||||
unsigned int md_len;
|
||||
void digest_file(char *filename, char *digest){
|
||||
int fd, i, n;
|
||||
unsigned char buf[MAXBUFSIZE], md[DIGEST_LENGTH];
|
||||
SHA256_CTX context;
|
||||
|
||||
md = EVP_get_digestbyname(digestname);
|
||||
if(md == NULL){
|
||||
syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
|
||||
return;
|
||||
}
|
||||
memset(digest, 0, 2*DIGEST_LENGTH+1);
|
||||
|
||||
fd = open(filename, O_RDONLY);
|
||||
if(fd == -1) return;
|
||||
|
||||
ctx = EVP_MD_CTX_new();
|
||||
EVP_DigestInit_ex(ctx, md, NULL);
|
||||
SHA256_Init(&context);
|
||||
|
||||
while((n = read(fd, buf, sizeof(buf))) > 0){
|
||||
EVP_DigestUpdate(ctx, buf, n);
|
||||
SHA256_Update(&context, buf, n);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
|
||||
EVP_DigestFinal_ex(ctx, md_value, &md_len);
|
||||
EVP_MD_CTX_free(ctx);
|
||||
SHA256_Final(md, &context);
|
||||
|
||||
for(i=0;i<DIGEST_LENGTH;i++)
|
||||
snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
|
||||
|
||||
}
|
||||
|
||||
|
||||
void digest_file(char *filename, char *digest){
|
||||
void digest_string(char *s, char *digest){
|
||||
int i;
|
||||
unsigned char md[DIGEST_LENGTH];
|
||||
|
||||
raw_digest_file("sha256", filename, &md[0]);
|
||||
SHA256_CTX context;
|
||||
|
||||
memset(digest, 0, 2*DIGEST_LENGTH+1);
|
||||
|
||||
for(int i=0;i<SHA256_DIGEST_LENGTH;i++){
|
||||
SHA256_Init(&context);
|
||||
|
||||
SHA256_Update(&context, s, strlen(s));
|
||||
|
||||
SHA256_Final(md, &context);
|
||||
|
||||
for(i=0;i<DIGEST_LENGTH;i++)
|
||||
snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
void digest_string(char *digestname, char *s, char *digest){
|
||||
EVP_MD_CTX *ctx;
|
||||
const EVP_MD *md;
|
||||
unsigned int i, md_len;
|
||||
unsigned char md_value[DIGEST_LENGTH];
|
||||
|
||||
memset(digest, 0, 2*DIGEST_LENGTH+2);
|
||||
|
||||
md = EVP_get_digestbyname(digestname);
|
||||
if(md == NULL){
|
||||
syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
|
||||
return;
|
||||
}
|
||||
|
||||
ctx = EVP_MD_CTX_new();
|
||||
EVP_DigestInit_ex(ctx, md, NULL);
|
||||
EVP_DigestUpdate(ctx, s, strlen(s));
|
||||
EVP_DigestFinal_ex(ctx, md_value, &md_len);
|
||||
EVP_MD_CTX_free(ctx);
|
||||
|
||||
for(i=0;i<md_len;i++){
|
||||
snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md_value[i]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void create_md5_from_email_address(char *puf, char *md5buf){
|
||||
digest_string("md5", puf, md5buf);
|
||||
md5buf[2*MD5_DIGEST_LENGTH] = ' ';
|
||||
}
|
||||
|
||||
49
src/dirs.c
49
src/dirs.c
@ -13,27 +13,11 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
void createdir(char *path, uid_t uid, gid_t gid, mode_t mode){
|
||||
struct stat st;
|
||||
|
||||
if(strlen(path) > 2){
|
||||
if(path[strlen(path)-1] == '/') path[strlen(path)-1] = '\0';
|
||||
|
||||
if(stat(path, &st)){
|
||||
if(mkdir(path, mode) == 0){
|
||||
if(chown(path, uid, gid))
|
||||
syslog(LOG_PRIORITY, "ERROR: createdir(): chown() failed on %s", path);
|
||||
syslog(LOG_PRIORITY, "created directory: *%s*", path);
|
||||
}
|
||||
else syslog(LOG_PRIORITY, "ERROR: could not create directory: *%s*", path);
|
||||
}
|
||||
}
|
||||
}
|
||||
void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
|
||||
|
||||
|
||||
void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid){
|
||||
void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
|
||||
char *p, s[SMALLBUFSIZE];
|
||||
int i;
|
||||
|
||||
p = strrchr(cfg->workdir, '/');
|
||||
if(p){
|
||||
@ -61,13 +45,24 @@ void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid){
|
||||
*p = '/';
|
||||
}
|
||||
|
||||
for(i=0; i<cfg->number_of_worker_processes; i++){
|
||||
snprintf(s, sizeof(s)-1, "%s/%d", cfg->workdir, i);
|
||||
#ifdef HAVE_ANTIVIRUS
|
||||
createdir(s, uid, gid, 0711);
|
||||
#else
|
||||
createdir(s, uid, gid, 0700);
|
||||
#endif
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
void createdir(char *path, uid_t uid, gid_t gid, mode_t mode){
|
||||
struct stat st;
|
||||
|
||||
if(strlen(path) > 2){
|
||||
if(path[strlen(path)-1] == '/') path[strlen(path)-1] = '\0';
|
||||
|
||||
if(stat(path, &st)){
|
||||
if(mkdir(path, mode) == 0){
|
||||
chown(path, uid, gid);
|
||||
syslog(LOG_PRIORITY, "created directory: *%s*", path);
|
||||
}
|
||||
else syslog(LOG_PRIORITY, "could not create directory: *%s*", path);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -15,12 +15,9 @@
|
||||
#define ERR_SETGID "ERR: setgid()"
|
||||
#define ERR_SELECT "ERR: select()"
|
||||
#define ERR_CHDIR "ERR: chdir() to working directory failed"
|
||||
#define ERR_DAEMON "ERR: daemon()"
|
||||
#define ERR_OPEN_TMP_FILE "ERR: opening a tempfile"
|
||||
#define ERR_TIMED_OUT "ERR: timed out"
|
||||
#define ERR_FORK_FAILED "ERR: cannot fork()"
|
||||
#define ERR_OPEN_DEDUP_FILE "ERR: cannot open dedup file"
|
||||
#define ERR_PID_FILE_EXISTS "ERR: pidfile exists. If piler daemon is not running, and the pidfile exists (unclean shutdown?) then remove it, and start piler again"
|
||||
|
||||
#define ERR_MYSQL_CONNECT "Cannot connect to mysql server"
|
||||
#define ERR_PSQL_CONNECT "Cannot connect to PSql server"
|
||||
@ -31,7 +28,4 @@
|
||||
|
||||
#define ERR_READING_KEY "ERR: reading cipher key"
|
||||
|
||||
#define ERR_LICENCE "ERR: licence validation error"
|
||||
#define ERR_LICENCE_IP "ERR: licenced IP-address doesn't match with listen_addr parameter"
|
||||
|
||||
#endif /* _ERRMSG_H */
|
||||
|
||||
289
src/extract.c
289
src/extract.c
@ -4,77 +4,70 @@
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/wait.h>
|
||||
#include <sys/stat.h>
|
||||
#include <dirent.h>
|
||||
#include <fcntl.h>
|
||||
#include <ctype.h>
|
||||
#include <signal.h>
|
||||
#include <piler.h>
|
||||
|
||||
#ifdef HAVE_ZIP
|
||||
#include <zip.h>
|
||||
#endif
|
||||
|
||||
#define die(e) do { syslog(LOG_INFO, "error: helper: %s", e); exit(EXIT_FAILURE); } while (0);
|
||||
|
||||
|
||||
int remove_xml(char *src, char *dest, int destlen, int *html){
|
||||
void remove_xml(char *buf, int *html){
|
||||
int i=0;
|
||||
char *p;
|
||||
|
||||
memset(dest, 0, destlen);
|
||||
p = buf;
|
||||
|
||||
for(; *src; src++){
|
||||
if(*src == '<'){
|
||||
*html = 1;
|
||||
for(; *p; p++){
|
||||
if(*p == '<'){ *html = 1; }
|
||||
|
||||
if(*html == 0){
|
||||
*(buf+i) = *p;
|
||||
i++;
|
||||
}
|
||||
else if(*src == '>'){
|
||||
|
||||
if(*p == '>'){
|
||||
*html = 0;
|
||||
// make sure there's a whitespace between tag contents
|
||||
if(i < destlen && i > 0 && !isspace(dest[i-1]))
|
||||
dest[i++] = ' ';
|
||||
}
|
||||
else{
|
||||
if(*html == 0){
|
||||
if(i < destlen) *(dest+i) = *src;
|
||||
i++;
|
||||
|
||||
if(i > 2 && *(buf+i-1) != ' '){
|
||||
*(buf+i) = ' '; i++;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return i;
|
||||
*(buf+i) = '\0';
|
||||
}
|
||||
|
||||
|
||||
#ifdef HAVE_ZIP
|
||||
int extract_opendocument(struct session_data *sdata, struct parser_state *state, char *filename, char *prefix){
|
||||
int errorp, i=0, html=0;
|
||||
unsigned int len2;
|
||||
char buf[4*MAXBUFSIZE], puf[4*MAXBUFSIZE];
|
||||
int extract_opendocument(struct session_data *sdata, struct _state *state, char *filename, char *prefix){
|
||||
int errorp, i=0, len=0, html=0;
|
||||
int len2;
|
||||
char buf[MAXBUFSIZE];
|
||||
struct zip *z;
|
||||
struct zip_stat sb;
|
||||
struct zip_file *zf;
|
||||
|
||||
z = zip_open(filename, ZIP_CHECKCONS, &errorp);
|
||||
if(!z){
|
||||
syslog(LOG_INFO, "%s: error: corrupt zip file=%s, error code=%d", sdata->ttmpfile, filename, errorp);
|
||||
return 1;
|
||||
}
|
||||
z = zip_open(filename, 0, &errorp);
|
||||
if(!z) return 1;
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
|
||||
while(zip_stat_index(z, i, 0, &sb) == 0){
|
||||
if(ZIP_EM_NONE == sb.encryption_method && strncmp(sb.name, prefix, strlen(prefix)) == 0 && (int)sb.size > 0){
|
||||
if(strncmp(sb.name, prefix, strlen(prefix)) == 0 && (int)sb.size > 0){
|
||||
|
||||
zf = zip_fopen_index(z, i, 0);
|
||||
if(zf){
|
||||
int len;
|
||||
while((len = zip_fread(zf, buf, sizeof(buf)-2)) > 0){
|
||||
|
||||
len2 = remove_xml(buf, puf, sizeof(puf), &html);
|
||||
remove_xml(buf, &html);
|
||||
len2 = strlen(buf);
|
||||
|
||||
if(len2 > 0 && state->bodylen < BIGBUFSIZE-len2-1){
|
||||
memcpy(&(state->b_body[state->bodylen]), puf, len2);
|
||||
memcpy(&(state->b_body[state->bodylen]), buf, len2);
|
||||
state->bodylen += len2;
|
||||
}
|
||||
|
||||
@ -97,8 +90,8 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,
|
||||
}
|
||||
|
||||
|
||||
int unzip_file(struct session_data *sdata, struct parser_state *state, char *filename, int *rec, struct config *cfg){
|
||||
int errorp, i=0, fd;
|
||||
int unzip_file(struct session_data *sdata, struct _state *state, char *filename, int *rec){
|
||||
int errorp, i=0, len=0, fd;
|
||||
char *p, extracted_filename[SMALLBUFSIZE], buf[MAXBUFSIZE];
|
||||
struct zip *z;
|
||||
struct zip_stat sb;
|
||||
@ -106,53 +99,41 @@ int unzip_file(struct session_data *sdata, struct parser_state *state, char *fil
|
||||
|
||||
(*rec)++;
|
||||
|
||||
z = zip_open(filename, ZIP_CHECKCONS, &errorp);
|
||||
if(!z){
|
||||
syslog(LOG_INFO, "%s: error: corrupt zip file=%s, error code=%d", sdata->ttmpfile, filename, errorp);
|
||||
return 1;
|
||||
}
|
||||
z = zip_open(filename, 0, &errorp);
|
||||
if(!z) return 1;
|
||||
|
||||
while(zip_stat_index(z, i, 0, &sb) == 0){
|
||||
//printf("processing file inside the zip: %s, index: %d, size: %d\n", sb.name, sb.index, (int)sb.size);
|
||||
|
||||
if(ZIP_EM_NONE == sb.encryption_method) {
|
||||
p = strrchr(sb.name, '.');
|
||||
|
||||
p = strrchr(sb.name, '.');
|
||||
if((int)sb.size > 0 && p && strcmp(get_attachment_extractor_by_filename((char*)sb.name), "other")){
|
||||
|
||||
if((int)sb.size > 0 && p && strcmp(get_attachment_extractor_by_filename((char*)sb.name), "other")){
|
||||
snprintf(extracted_filename, sizeof(extracted_filename)-1, "%s-%d-%d%s", sdata->ttmpfile, *rec, i, p);
|
||||
|
||||
snprintf(extracted_filename, sizeof(extracted_filename)-1, "%s-%d-%d%s", sdata->ttmpfile, *rec, i, p);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: writing zip content to '%s'", sdata->ttmpfile, extracted_filename);
|
||||
|
||||
fd = open(extracted_filename, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
|
||||
if(fd != -1){
|
||||
zf = zip_fopen_index(z, i, 0);
|
||||
if(zf){
|
||||
int len;
|
||||
while((len = zip_fread(zf, buf, sizeof(buf))) > 0){
|
||||
if(write(fd, buf, len) == -1) syslog(LOG_PRIORITY, "ERROR: error writing to fd in %s", __func__);
|
||||
}
|
||||
zip_fclose(zf);
|
||||
fd = open(extracted_filename, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
|
||||
if(fd != -1){
|
||||
zf = zip_fopen_index(z, i, 0);
|
||||
if(zf){
|
||||
while((len = zip_fread(zf, buf, sizeof(buf))) > 0){
|
||||
write(fd, buf, len);
|
||||
}
|
||||
else syslog(LOG_PRIORITY, "%s: cannot extract '%s' from '%s'", sdata->ttmpfile, sb.name, extracted_filename);
|
||||
|
||||
close(fd);
|
||||
|
||||
extract_attachment_content(sdata, state, extracted_filename, get_attachment_extractor_by_filename(extracted_filename), rec, cfg);
|
||||
|
||||
unlink(extracted_filename);
|
||||
|
||||
}
|
||||
else {
|
||||
syslog(LOG_PRIORITY, "%s: cannot open '%s'", sdata->ttmpfile, extracted_filename);
|
||||
zip_fclose(zf);
|
||||
}
|
||||
else syslog(LOG_PRIORITY, "%s: cannot extract '%s' from '%s'", sdata->ttmpfile, sb.name, extracted_filename);
|
||||
|
||||
close(fd);
|
||||
|
||||
extract_attachment_content(sdata, state, extracted_filename, get_attachment_extractor_by_filename(extracted_filename), rec);
|
||||
|
||||
unlink(extracted_filename);
|
||||
|
||||
}
|
||||
else {
|
||||
syslog(LOG_PRIORITY, "%s: cannot open '%s'", sdata->ttmpfile, extracted_filename);
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
syslog(LOG_PRIORITY, "error: attachment ('%s') is in encrypted zip file", sb.name);
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
@ -165,61 +146,66 @@ int unzip_file(struct session_data *sdata, struct parser_state *state, char *fil
|
||||
|
||||
#endif
|
||||
|
||||
void read_content_with_popen(struct session_data *sdata, struct _state *state, char *cmd){
|
||||
int len;
|
||||
char buf[MAXBUFSIZE];
|
||||
FILE *f;
|
||||
|
||||
#ifdef HAVE_TNEF
|
||||
f = popen(cmd, "r");
|
||||
if(f){
|
||||
while(fgets(buf, sizeof(buf)-1, f)){
|
||||
len = strlen(buf);
|
||||
|
||||
int extract_tnef(struct session_data *sdata, struct parser_state *state, char *filename, struct config *cfg){
|
||||
int rc=0, n, rec=1;
|
||||
char tmpdir[BUFLEN], buf[SMALLBUFSIZE];
|
||||
struct dirent **namelist;
|
||||
|
||||
memset(tmpdir, 0, sizeof(tmpdir));
|
||||
make_random_string((unsigned char *)&tmpdir[0], sizeof(tmpdir)-3);
|
||||
|
||||
memcpy(&tmpdir[sizeof(tmpdir)-3], ".d", 2);
|
||||
|
||||
if(mkdir(tmpdir, 0700)) return rc;
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "%s --unix-paths -C %s %s", HAVE_TNEF, tmpdir, filename);
|
||||
|
||||
if(system(buf) == -1) syslog(LOG_INFO, "error: running %s", buf);
|
||||
|
||||
n = scandir(tmpdir, &namelist, NULL, alphasort);
|
||||
if(n < 0) syslog(LOG_INFO, "error: reading %s", tmpdir);
|
||||
else {
|
||||
while(n--){
|
||||
if(strcmp(namelist[n]->d_name, ".") && strcmp(namelist[n]->d_name, "..")){
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "%s/%s", tmpdir, namelist[n]->d_name);
|
||||
extract_attachment_content(sdata, state, buf, get_attachment_extractor_by_filename(buf), &rec, cfg);
|
||||
|
||||
unlink(buf);
|
||||
if(state->bodylen < BIGBUFSIZE-len-1){
|
||||
memcpy(&(state->b_body[state->bodylen]), buf, len);
|
||||
state->bodylen += len;
|
||||
}
|
||||
|
||||
free(namelist[n]);
|
||||
else break;
|
||||
}
|
||||
free(namelist);
|
||||
|
||||
fclose(f);
|
||||
}
|
||||
else syslog(LOG_PRIORITY, "%s: popen(): %s", sdata->ttmpfile, buf);
|
||||
|
||||
rmdir(tmpdir);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
void extract_attachment_content(struct session_data *sdata, struct _state *state, char *filename, char *type, int *rec){
|
||||
char cmd[SMALLBUFSIZE];
|
||||
|
||||
if(strcmp(type, "other") == 0) return;
|
||||
|
||||
memset(cmd, 0, sizeof(cmd));
|
||||
|
||||
#ifdef HAVE_PDFTOTEXT
|
||||
if(strcmp(type, "pdf") == 0) snprintf(cmd, sizeof(cmd)-1, "%s -enc UTF-8 %s -", HAVE_PDFTOTEXT, filename);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_CATDOC
|
||||
if(strcmp(type, "doc") == 0) snprintf(cmd, sizeof(cmd)-1, "%s -d utf-8 %s", HAVE_CATDOC, filename);
|
||||
#endif
|
||||
|
||||
void kill_helper(){
|
||||
syslog(LOG_PRIORITY, "error: helper is killed by alarm");
|
||||
die("timeout for helper!");
|
||||
}
|
||||
#ifdef HAVE_CATPPT
|
||||
if(strcmp(type, "ppt") == 0) snprintf(cmd, sizeof(cmd)-1, "%s -d utf-8 %s", HAVE_CATPPT, filename);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_XLS2CSV
|
||||
if(strcmp(type, "xls") == 0) snprintf(cmd, sizeof(cmd)-1, "%s -d utf-8 %s", HAVE_XLS2CSV, filename);
|
||||
#endif
|
||||
|
||||
void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg){
|
||||
int link[2];
|
||||
pid_t pid;
|
||||
#ifdef HAVE_PPTHTML
|
||||
if(strcmp(type, "ppt") == 0) snprintf(cmd, sizeof(cmd)-1, "%s %s", HAVE_PPTHTML, filename);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_UNRTF
|
||||
if(strcmp(type, "rtf") == 0) snprintf(cmd, sizeof(cmd)-1, "%s --text %s", HAVE_UNRTF, filename);
|
||||
#endif
|
||||
|
||||
if(strlen(cmd) > 12){
|
||||
read_content_with_popen(sdata, state, cmd);
|
||||
return;
|
||||
}
|
||||
|
||||
if(strcmp(type, "other") == 0 || strcmp(type, "text") == 0) return;
|
||||
|
||||
#ifdef HAVE_ZIP
|
||||
if(strcmp(type, "odf") == 0){
|
||||
@ -244,7 +230,7 @@ void extract_attachment_content(struct session_data *sdata, struct parser_state
|
||||
|
||||
if(strcmp(type, "zip") == 0){
|
||||
if(*rec < MAX_ZIP_RECURSION_LEVEL){
|
||||
unzip_file(sdata, state, filename, rec, cfg);
|
||||
unzip_file(sdata, state, filename, rec);
|
||||
}
|
||||
else {
|
||||
syslog(LOG_PRIORITY, "%s: multiple recursion level zip attachment, skipping %s", sdata->ttmpfile, filename);
|
||||
@ -252,79 +238,6 @@ void extract_attachment_content(struct session_data *sdata, struct parser_state
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_TNEF
|
||||
if(strcmp(type, "tnef") == 0){
|
||||
extract_tnef(sdata, state, filename, cfg);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
* http://stackoverflow.com/questions/7292642/grabbing-output-from-exec
|
||||
*/
|
||||
|
||||
if(pipe(link) == -1){
|
||||
syslog(LOG_PRIORITY, "%s: cannot open link", sdata->ttmpfile);
|
||||
return;
|
||||
}
|
||||
|
||||
if((pid = fork()) == -1){
|
||||
syslog(LOG_PRIORITY, "%s: cannot fork", sdata->ttmpfile);
|
||||
close(link[0]);
|
||||
close(link[1]);
|
||||
return;
|
||||
}
|
||||
|
||||
if(pid == 0){
|
||||
dup2(link[1], STDOUT_FILENO);
|
||||
close(link[0]);
|
||||
close(link[1]);
|
||||
|
||||
alarm(cfg->helper_timeout);
|
||||
sig_catch(SIGALRM, kill_helper);
|
||||
|
||||
#ifdef HAVE_PDFTOTEXT
|
||||
if(strcmp(type, "pdf") == 0) execl(HAVE_PDFTOTEXT, HAVE_PDFTOTEXT, "-enc", "UTF-8", filename, "-", (char *) 0);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_CATDOC
|
||||
if(strcmp(type, "doc") == 0) execl(HAVE_CATDOC, HAVE_CATDOC, "-d", "utf-8", filename, (char *) 0);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_CATPPT
|
||||
if(strcmp(type, "ppt") == 0) execl(HAVE_CATPPT, HAVE_CATPPT, "-d", "utf-8", filename, (char *) 0);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_XLS2CSV
|
||||
if(strcmp(type, "xls") == 0) execl(HAVE_XLS2CSV, HAVE_XLS2CSV, "-d", "utf-8", filename, (char *) 0);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_PPTHTML
|
||||
if(strcmp(type, "ppt") == 0) execl(HAVE_PPTHTML, HAVE_PPTHTML, filename, (char *) 0);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_UNRTF
|
||||
if(strcmp(type, "rtf") == 0) execl(HAVE_UNRTF, HAVE_UNRTF, "--text", filename, (char *) 0);
|
||||
#endif
|
||||
|
||||
die("execl");
|
||||
}
|
||||
else {
|
||||
close(link[1]);
|
||||
ssize_t n;
|
||||
char outbuf[MAXBUFSIZE];
|
||||
|
||||
while((n = read(link[0], outbuf, sizeof(outbuf))) > 0){
|
||||
if(state->bodylen < BIGBUFSIZE-n-1){
|
||||
memcpy(&(state->b_body[state->bodylen]), outbuf, n);
|
||||
state->bodylen += n;
|
||||
}
|
||||
}
|
||||
close(link[0]);
|
||||
wait(NULL);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
163
src/hash.c
163
src/hash.c
@ -1,163 +0,0 @@
|
||||
/*
|
||||
* hash.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
inline int hash(unsigned int key){
|
||||
return key % MAXHASH;
|
||||
}
|
||||
|
||||
|
||||
void inithash(struct node *xhash[]){
|
||||
int i;
|
||||
|
||||
for(i=0;i<MAXHASH;i++){
|
||||
xhash[i] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void clearhash(struct node *xhash[]){
|
||||
int i;
|
||||
struct node *p, *q;
|
||||
|
||||
for(i=0;i<MAXHASH;i++){
|
||||
q = xhash[i];
|
||||
while(q != NULL){
|
||||
p = q;
|
||||
|
||||
q = q->r;
|
||||
if(p->str){
|
||||
free(p->str);
|
||||
}
|
||||
free(p);
|
||||
}
|
||||
xhash[i] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
struct node *makenewnode(char *s){
|
||||
struct node *h;
|
||||
int len;
|
||||
|
||||
if(s == NULL) return NULL;
|
||||
|
||||
len = strlen(s);
|
||||
|
||||
if((h = malloc(sizeof(struct node))) == NULL) return NULL;
|
||||
|
||||
memset(h, 0, sizeof(struct node));
|
||||
|
||||
h->str = malloc(len+2);
|
||||
|
||||
if(h->str == NULL){
|
||||
free(h);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(h->str, 0, len+2);
|
||||
|
||||
snprintf(h->str, len+1, "%s", s);
|
||||
|
||||
h->key = DJBHash(s, len);
|
||||
h->r = NULL;
|
||||
|
||||
return h;
|
||||
}
|
||||
|
||||
|
||||
int addnode(struct node *xhash[], char *s){
|
||||
struct node *p=NULL, *q;
|
||||
unsigned int key = 0;
|
||||
int len;
|
||||
|
||||
if(s == NULL) return 0;
|
||||
|
||||
len = strlen(s);
|
||||
|
||||
key = DJBHash(s, len);
|
||||
|
||||
if(xhash[hash(key)] == NULL){
|
||||
xhash[hash(key)] = makenewnode(s);
|
||||
}
|
||||
else {
|
||||
q = xhash[hash(key)];
|
||||
while(q != NULL){
|
||||
p = q;
|
||||
if(p->key == key){
|
||||
return 0;
|
||||
}
|
||||
else {
|
||||
q = q->r;
|
||||
}
|
||||
}
|
||||
if(p) p->r = makenewnode(s);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
struct node *findnode(struct node *xhash[], char *s){
|
||||
struct node *q;
|
||||
unsigned int key;
|
||||
int len;
|
||||
|
||||
if(s == NULL) return NULL;
|
||||
|
||||
len = strlen(s);
|
||||
|
||||
key = DJBHash(s, len);
|
||||
|
||||
q = xhash[hash(key)];
|
||||
|
||||
if(q == NULL) return NULL;
|
||||
|
||||
while(q != NULL){
|
||||
|
||||
if(strcmp(q->str, s) == 0){
|
||||
return q;
|
||||
}
|
||||
else {
|
||||
q = q->r;
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
int is_substr_in_hash(struct node *xhash[], char *s){
|
||||
int i;
|
||||
struct node *q;
|
||||
|
||||
for(i=0;i<MAXHASH;i++){
|
||||
q = xhash[i];
|
||||
while(q != NULL){
|
||||
if(q->str && strstr(s, q->str)) return 1;
|
||||
|
||||
q = q->r;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
unsigned int DJBHash(char* str, unsigned int len){
|
||||
unsigned int hashval = 5381;
|
||||
unsigned int i = 0;
|
||||
|
||||
for(i=0; i < len; str++, i++){
|
||||
hashval = ((hashval << 5) + hashval) + (*str);
|
||||
}
|
||||
|
||||
return hashval;
|
||||
}
|
||||
20
src/hash.h
20
src/hash.h
@ -1,20 +0,0 @@
|
||||
/*
|
||||
* hash.h, SJ
|
||||
*/
|
||||
|
||||
#ifndef _HASH_H
|
||||
#define _HASH_H
|
||||
|
||||
#include "cfg.h"
|
||||
#include "defs.h"
|
||||
|
||||
|
||||
void inithash(struct node *xhash[]);
|
||||
void clearhash(struct node *xhash[]);
|
||||
struct node *makenewnode(char *s);
|
||||
int addnode(struct node *xhash[], char *s);
|
||||
struct node *findnode(struct node *xhash[], char *s);
|
||||
int is_substr_in_hash(struct node *xhash[], char *s);
|
||||
unsigned int DJBHash(char* str, unsigned int len);
|
||||
|
||||
#endif /* _HASH_H */
|
||||
38
src/html.h
Normal file
38
src/html.h
Normal file
@ -0,0 +1,38 @@
|
||||
|
||||
struct html_tag {
|
||||
unsigned char length;
|
||||
char *entity;
|
||||
};
|
||||
|
||||
#define NUM_OF_SKIP_TAGS2 10
|
||||
|
||||
struct html_tag skip_html_tags2[] = {
|
||||
{ 4, "html" },
|
||||
{ 5, "/html" },
|
||||
{ 5, "/body" },
|
||||
{ 4, "meta" },
|
||||
{ 4, "head" },
|
||||
{ 5, "/head" },
|
||||
{ 5, "style" },
|
||||
{ 6, "/style" },
|
||||
{ 3, "div" },
|
||||
{ 4, "/div" }
|
||||
};
|
||||
|
||||
|
||||
#define NUM_OF_SKIP_TAGS 11
|
||||
|
||||
struct html_tag skip_html_tags[] = {
|
||||
{ 5, "style" },
|
||||
{ 4, "dir=" },
|
||||
{ 8, "content=" },
|
||||
{ 5, "name=" },
|
||||
{ 3, "id=" },
|
||||
{ 2, "v:" },
|
||||
{ 6, "class=" },
|
||||
{ 5, "xmlns" },
|
||||
{ 10, "http-equiv" },
|
||||
{ 7, "spidmax" },
|
||||
{ 5, "data=" }
|
||||
};
|
||||
|
||||
@ -285,3 +285,4 @@ struct mi htmlentities[] = {
|
||||
|
||||
|
||||
#endif /* _HTMLENTITIES_H */
|
||||
|
||||
|
||||
580
src/imap.c
580
src/imap.c
@ -23,11 +23,12 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int get_message_length_from_imap_answer(char *s){
|
||||
int get_message_length_from_imap_answer(char *s, int *_1st_line_bytes){
|
||||
char *p, *q;
|
||||
int len=0;
|
||||
|
||||
p = strstr(s, "\r");
|
||||
|
||||
p = strstr(s, "\r\n");
|
||||
if(!p){
|
||||
printf("invalid reply: %s", s);
|
||||
return len;
|
||||
@ -35,6 +36,9 @@ int get_message_length_from_imap_answer(char *s){
|
||||
|
||||
*p = '\0';
|
||||
|
||||
//*_1st_line_bytes = strlen(s)+2;
|
||||
*_1st_line_bytes = p-s+2;
|
||||
|
||||
if(*(p-1) == '}') *(p-1) = '\0';
|
||||
|
||||
|
||||
@ -51,107 +55,39 @@ int get_message_length_from_imap_answer(char *s){
|
||||
}
|
||||
|
||||
|
||||
int read_response(char *buf, int buflen, struct data *data){
|
||||
int i=0, n, len=0, rc=0;
|
||||
char puf[MAXBUFSIZE], tagok[SMALLBUFSIZE], tagno[SMALLBUFSIZE], tagbad[SMALLBUFSIZE];
|
||||
int is_last_complete_packet(char *s, int len, char *tagok, char *tagbad, int *pos){
|
||||
char *p;
|
||||
|
||||
snprintf(tagok, sizeof(tagok)-1, "A%d OK", data->import->seq);
|
||||
snprintf(tagno, sizeof(tagno)-1, "A%d NO", data->import->seq);
|
||||
snprintf(tagbad, sizeof(tagbad)-1, "A%d BAD", data->import->seq);
|
||||
*pos = 0;
|
||||
|
||||
memset(buf, 0, buflen);
|
||||
|
||||
while(!strstr(buf, tagok)){
|
||||
n = recvtimeoutssl(data->net, puf, sizeof(puf));
|
||||
|
||||
if(n + len < buflen) strncat(buf, puf, n);
|
||||
else goto END;
|
||||
|
||||
/*
|
||||
* possible error message from the imap server:
|
||||
*
|
||||
* * BYE Temporary problem, please try again later\r\n
|
||||
*/
|
||||
|
||||
if(i == 0 && (strstr(puf, tagno) || strstr(puf, tagbad) || strstr(puf, "* BYE ")) ) goto END;
|
||||
|
||||
|
||||
len += n;
|
||||
i++;
|
||||
if(*(s+len-2) == '\r' && *(s+len-1) == '\n'){
|
||||
if((p = strstr(s, tagok))){
|
||||
*pos = p - s;
|
||||
if(*pos > 3) *pos -= 2;
|
||||
return 1;
|
||||
}
|
||||
if(strstr(s, tagbad)) return 1;
|
||||
}
|
||||
|
||||
rc = 1;
|
||||
|
||||
END:
|
||||
|
||||
(data->import->seq)++;
|
||||
|
||||
return rc;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int connect_to_imap_server(struct data *data){
|
||||
char buf[MAXBUFSIZE];
|
||||
int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg){
|
||||
int rc=ERR, i, n, pos, endpos, messages=0, len, readlen, fd, lastpos, nreads, processed_messages=0;
|
||||
char *p, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE], buf[MAXBUFSIZE], filename[SMALLBUFSIZE];
|
||||
char aggrbuf[3*MAXBUFSIZE];
|
||||
|
||||
data->import->cap_uidplus = 0;
|
||||
snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "\r\nA%d OK", (*seq)++);
|
||||
snprintf(buf, sizeof(buf)-1, "%s SELECT \"%s\"\r\n", tag, folder);
|
||||
|
||||
if(data->net->use_ssl == 1){
|
||||
init_ssl_to_server(data);
|
||||
}
|
||||
n = write1(sd, buf, use_ssl, data->ssl);
|
||||
n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
|
||||
|
||||
|
||||
recvtimeoutssl(data->net, buf, sizeof(buf));
|
||||
|
||||
|
||||
/* imap cmd: LOGIN */
|
||||
|
||||
if(strcmp(data->import->username, "ZIMBRA") == 0){
|
||||
snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
|
||||
}
|
||||
else {
|
||||
snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
|
||||
}
|
||||
|
||||
write1(data->net, buf, strlen(buf));
|
||||
if(read_response(buf, sizeof(buf), data) == 0){
|
||||
printf("login failed, server reponse: %s\n", buf);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(strstr(buf, "UIDPLUS")){
|
||||
data->import->cap_uidplus = 1;
|
||||
}
|
||||
else {
|
||||
|
||||
/* run the CAPABILITY command if the reply doesn't contain the UIDPLUS capability */
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "A%d CAPABILITY\r\n", data->import->seq);
|
||||
|
||||
write1(data->net, buf, strlen(buf));
|
||||
read_response(buf, sizeof(buf), data);
|
||||
|
||||
if(strstr(buf, "UIDPLUS")) data->import->cap_uidplus = 1;
|
||||
}
|
||||
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
int imap_select_cmd_on_folder(char *folder, struct data *data){
|
||||
int messages=0;
|
||||
char *p, buf[MAXBUFSIZE];
|
||||
|
||||
if(strchr(folder, '"'))
|
||||
snprintf(buf, sizeof(buf)-1, "A%d SELECT %s\r\n", data->import->seq, folder);
|
||||
else
|
||||
snprintf(buf, sizeof(buf)-1, "A%d SELECT \"%s\"\r\n", data->import->seq, folder);
|
||||
|
||||
write1(data->net, buf, strlen(buf));
|
||||
if(read_response(buf, sizeof(buf), data) == 0){
|
||||
if(!strstr(buf, tagok)){
|
||||
trimBuffer(buf);
|
||||
printf("ERROR: select cmd error: %s\n", buf);
|
||||
return messages;
|
||||
printf("error: %s\n", buf);
|
||||
return rc;
|
||||
}
|
||||
|
||||
p = strstr(buf, " EXISTS");
|
||||
@ -164,343 +100,229 @@ int imap_select_cmd_on_folder(char *folder, struct data *data){
|
||||
}
|
||||
}
|
||||
|
||||
if(data->quiet == 0){printf("found %d messages\n", messages); }
|
||||
printf("found %d messages\n", messages);
|
||||
|
||||
data->import->total_messages += messages;
|
||||
if(messages <= 0) return rc;
|
||||
|
||||
return messages;
|
||||
}
|
||||
for(i=1; i<=messages; i++){
|
||||
processed_messages++;
|
||||
printf("processed: %7d\r", processed_messages); fflush(stdout);
|
||||
|
||||
snprintf(tag, sizeof(tag)-1, "A%d", *seq);
|
||||
snprintf(tagok, sizeof(tagok)-1, "\r\nA%d OK", (*seq)++);
|
||||
snprintf(tagbad, sizeof(tagbad)-1, "\r\n%s BAD", tag);
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "%s FETCH %d (BODY.PEEK[])\r\n", tag, i);
|
||||
|
||||
snprintf(filename, sizeof(filename)-1, "%s-%d.txt", folder, i);
|
||||
unlink(filename);
|
||||
|
||||
fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
|
||||
if(fd == -1){
|
||||
printf("cannot open: %s\n", filename);
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int imap_download_email(struct data *data, int i){
|
||||
int fd, len, result, tagoklen, tagbadlen;
|
||||
int n, readlen=0, nreads=0, readpos=0, finished=0, msglen=0, msg_written_len=0;
|
||||
char *p, buf[MAXBUFSIZE], puf[MAXBUFSIZE], tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE];
|
||||
n = write1(sd, buf, use_ssl, data->ssl);
|
||||
|
||||
data->import->processed_messages++;
|
||||
readlen = 0;
|
||||
pos = 0;
|
||||
len = 0;
|
||||
nreads = 0;
|
||||
endpos = 0;
|
||||
|
||||
snprintf(data->import->filename, SMALLBUFSIZE-1, "%d-imap-%d.txt", getpid(), data->import->processed_messages);
|
||||
memset(aggrbuf, 0, sizeof(aggrbuf));
|
||||
lastpos = 0;
|
||||
|
||||
unlink(data->import->filename);
|
||||
|
||||
fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
|
||||
if(fd == -1){
|
||||
printf("cannot open: %s\n", data->import->filename);
|
||||
return ERR;
|
||||
}
|
||||
while((n = recvtimeoutssl(sd, buf, sizeof(buf), 15, use_ssl, data->ssl)) > 0){
|
||||
nreads++;
|
||||
readlen += n;
|
||||
|
||||
snprintf(tag, sizeof(tag)-1, "A%d", data->import->seq);
|
||||
snprintf(tagok, sizeof(tagok)-1, "A%d OK", (data->import->seq)++);
|
||||
snprintf(tagbad, sizeof(tagbad)-1, "%s BAD", tag);
|
||||
if(nreads == 1){
|
||||
len = get_message_length_from_imap_answer(buf, &pos);
|
||||
|
||||
tagoklen = strlen(tagok);
|
||||
tagbadlen = strlen(tagbad);
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "%s FETCH %d (BODY.PEEK[])\r\n", tag, i);
|
||||
write1(data->net, buf, strlen(buf));
|
||||
|
||||
while((n = recvtimeoutssl(data->net, &buf[readpos], sizeof(buf)-readpos)) > 0){
|
||||
|
||||
readlen += n;
|
||||
|
||||
if(strchr(buf, '\n')){
|
||||
readpos = 0;
|
||||
p = &buf[0];
|
||||
do {
|
||||
nreads++;
|
||||
memset(puf, 0, sizeof(puf));
|
||||
p = split(p, '\n', puf, sizeof(puf)-1, &result);
|
||||
len = strlen(puf);
|
||||
|
||||
if(result == 1){
|
||||
// process a complete line
|
||||
|
||||
if(nreads == 1){
|
||||
|
||||
if(strcasestr(puf, " FETCH ")){
|
||||
msglen = get_message_length_from_imap_answer(puf);
|
||||
|
||||
if(msglen == 0){
|
||||
finished = 1;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if(strcasestr(puf, " BYE")){
|
||||
printf("imap server sent BYE response: '%s'\n", puf);
|
||||
close(fd);
|
||||
unlink(data->import->filename);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(len > 0 && msg_written_len < msglen){
|
||||
if(write(fd, puf, len) == -1) printf("ERROR: writing to fd\n");
|
||||
if(write(fd, "\n", 1) == -1) printf("ERROR: writing to fd\n");
|
||||
msg_written_len += len + 1;
|
||||
}
|
||||
|
||||
if(strncmp(puf, tagok, tagoklen) == 0){
|
||||
finished = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
if(strncmp(puf, tagbad, tagbadlen) == 0){
|
||||
printf("ERROR happened reading the message!\n");
|
||||
finished = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
// prepend the last incomplete line back to 'buf'
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "%s", puf);
|
||||
readpos = len;
|
||||
if(len < 10){
|
||||
printf("%d: too short message! %s\n", i, buf);
|
||||
break;
|
||||
}
|
||||
|
||||
} while(p);
|
||||
|
||||
|
||||
}
|
||||
else {
|
||||
readpos += n;
|
||||
}
|
||||
|
||||
if(finished == 1) break;
|
||||
}
|
||||
|
||||
close(fd);
|
||||
|
||||
if(msglen > 10) return OK;
|
||||
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
void imap_delete_message(struct data *data, int i){
|
||||
char buf[SMALLBUFSIZE];
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", data->import->seq, i);
|
||||
write1(data->net, buf, strlen(buf));
|
||||
read_response(buf, sizeof(buf), data);
|
||||
}
|
||||
|
||||
|
||||
void imap_move_message_to_folder(struct data *data, int i){
|
||||
int tagoklen;
|
||||
char buf[SMALLBUFSIZE], tagok[SMALLBUFSIZE];
|
||||
|
||||
snprintf(tagok, sizeof(tagok)-1, "A%d OK", data->import->seq);
|
||||
tagoklen = strlen(tagok);
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "A%d COPY %d %s\r\n", data->import->seq, i, data->import->move_folder);
|
||||
write1(data->net, buf, strlen(buf));
|
||||
read_response(buf, sizeof(buf), data);
|
||||
|
||||
if(strncmp(buf, tagok, tagoklen) == 0){
|
||||
snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", data->import->seq, i);
|
||||
write1(data->net, buf, strlen(buf));
|
||||
read_response(buf, sizeof(buf), data);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void imap_expunge_message(struct data *data){
|
||||
char buf[SMALLBUFSIZE];
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "A%d EXPUNGE\r\n", data->import->seq);
|
||||
write1(data->net, buf, strlen(buf));
|
||||
read_response(buf, sizeof(buf), data);
|
||||
}
|
||||
|
||||
|
||||
int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
int i, messages=0;
|
||||
|
||||
messages = imap_select_cmd_on_folder(folder, data);
|
||||
|
||||
if(messages <= 0) return OK;
|
||||
|
||||
if(data->recursive_folder_names == 1){
|
||||
data->folder = get_folder_id(sdata, folder, 0);
|
||||
if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, folder, 0);
|
||||
}
|
||||
|
||||
for(i=data->import->start_position; i<=messages; i++){
|
||||
if(imap_download_email(data, i) == OK){
|
||||
if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
|
||||
|
||||
if(data->import->dryrun == 0){
|
||||
int rc = import_message(sdata, data, counters, cfg);
|
||||
|
||||
if(data->import->remove_after_import == 1 && rc == OK){
|
||||
imap_delete_message(data, i);
|
||||
}
|
||||
|
||||
if(data->import->move_folder && data->import->cap_uidplus == 1){
|
||||
imap_move_message_to_folder(data, i);
|
||||
}
|
||||
}
|
||||
|
||||
if(data->import->download_only == 0) unlink(data->import->filename);
|
||||
}
|
||||
if(lastpos + 1 + n < sizeof(aggrbuf)){
|
||||
|
||||
/* whether to quit after processing a batch of messages */
|
||||
if(nreads == 1){
|
||||
memcpy(aggrbuf+lastpos, buf+pos, n-pos);
|
||||
lastpos += n-pos;
|
||||
}
|
||||
else {
|
||||
memcpy(aggrbuf+lastpos, buf, n);
|
||||
lastpos += n;
|
||||
}
|
||||
}
|
||||
else {
|
||||
write(fd, aggrbuf, sizeof(buf));
|
||||
|
||||
if(data->import->batch_processing_limit > 0 && data->import->processed_messages >= data->import->batch_processing_limit){
|
||||
break;
|
||||
}
|
||||
memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
|
||||
lastpos -= sizeof(buf);
|
||||
|
||||
memcpy(aggrbuf+lastpos, buf, n);
|
||||
lastpos += n;
|
||||
}
|
||||
|
||||
if(is_last_complete_packet(aggrbuf, lastpos, tagok, tagbad, &endpos) == 1){
|
||||
write(fd, aggrbuf, lastpos-(lastpos-endpos));
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
close(fd);
|
||||
|
||||
rc = import_message(filename, sdata, data, cfg);
|
||||
|
||||
if(rc == ERR) printf("error importing '%s'\n", filename);
|
||||
else unlink(filename);
|
||||
}
|
||||
|
||||
if((data->import->remove_after_import == 1 || data->import->move_folder) && data->import->dryrun == 0){
|
||||
imap_expunge_message(data);
|
||||
}
|
||||
|
||||
|
||||
if(data->quiet == 0){printf("\n"); }
|
||||
printf("\n");
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
void send_imap_close(struct data *data){
|
||||
char puf[SMALLBUFSIZE];
|
||||
snprintf(puf, sizeof(puf)-1, "A%d CLOSE\r\n", data->import->seq);
|
||||
int connect_to_imap_server(int sd, int *seq, char *username, char *password, int port, struct __data *data, int use_ssl){
|
||||
int n;
|
||||
char tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], buf[MAXBUFSIZE];
|
||||
X509* server_cert;
|
||||
char *str;
|
||||
|
||||
write1(data->net, puf, strlen(puf));
|
||||
|
||||
if(use_ssl == 1){
|
||||
|
||||
SSL_library_init();
|
||||
SSL_load_error_strings();
|
||||
|
||||
data->ctx = SSL_CTX_new(SSLv3_client_method());
|
||||
CHK_NULL(data->ctx, "internal SSL error");
|
||||
|
||||
data->ssl = SSL_new(data->ctx);
|
||||
CHK_NULL(data->ssl, "internal ssl error");
|
||||
|
||||
SSL_set_fd(data->ssl, sd);
|
||||
n = SSL_connect(data->ssl);
|
||||
CHK_SSL(n, "internal ssl error");
|
||||
|
||||
//printf("Cipher: %s\n", SSL_get_cipher(data->ssl));
|
||||
|
||||
server_cert = SSL_get_peer_certificate(data->ssl);
|
||||
CHK_NULL(server_cert, "server cert error");
|
||||
|
||||
//if(verbose){
|
||||
str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
|
||||
CHK_NULL(str, "error in server cert");
|
||||
printf("server cert:\n\t subject: %s\n", str);
|
||||
OPENSSL_free(str);
|
||||
|
||||
str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
|
||||
CHK_NULL(str, "error in server cert");
|
||||
printf("\t issuer: %s\n\n", str);
|
||||
OPENSSL_free(str);
|
||||
//}
|
||||
|
||||
X509_free(server_cert);
|
||||
}
|
||||
|
||||
|
||||
n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
|
||||
|
||||
|
||||
snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
|
||||
snprintf(buf, sizeof(buf)-1, "%s CAPABILITY\r\n", tag);
|
||||
|
||||
write1(sd, buf, use_ssl, data->ssl);
|
||||
n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
|
||||
|
||||
|
||||
snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
|
||||
snprintf(buf, sizeof(buf)-1, "%s LOGIN %s \"%s\"\r\n", tag, username, password);
|
||||
|
||||
write1(sd, buf, use_ssl, data->ssl);
|
||||
n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
|
||||
|
||||
if(strncmp(buf, tagok, strlen(tagok))){
|
||||
printf("login failed, server reponse: %s\n", buf);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
int list_folders(struct data *data){
|
||||
char *p, *q, *r, *buf, *ruf, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], puf[MAXBUFSIZE];
|
||||
char attrs[SMALLBUFSIZE], folder[SMALLBUFSIZE];
|
||||
int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result;
|
||||
void close_connection(int sd, struct __data *data, int use_ssl){
|
||||
close(sd);
|
||||
|
||||
if(use_ssl == 1){
|
||||
SSL_shutdown(data->ssl);
|
||||
SSL_free(data->ssl);
|
||||
SSL_CTX_free(data->ctx);
|
||||
ERR_free_strings();
|
||||
}
|
||||
}
|
||||
|
||||
if(data->quiet == 0){printf("List of IMAP folders:\n"); }
|
||||
|
||||
buf = malloc(len);
|
||||
if(!buf) return rc;
|
||||
int list_folders(int sd, int *seq, char *folders, int foldersize, int use_ssl, struct __data *data){
|
||||
char *p, *q, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], buf[3*MAXBUFSIZE+3], puf[MAXBUFSIZE];
|
||||
int len=0, n;
|
||||
|
||||
memset(buf, 0, len);
|
||||
memset(buf, 0, sizeof(buf));
|
||||
|
||||
snprintf(tag, sizeof(tag)-1, "A%d", data->import->seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (data->import->seq)++);
|
||||
if(data->import->folder_imap == NULL)
|
||||
snprintf(puf, sizeof(puf)-1, "%s LIST \"\" \"*\"\r\n", tag);
|
||||
else
|
||||
snprintf(puf, sizeof(puf)-1, "%s LIST \"%s\" \"*\"\r\n", tag, data->import->folder_imap);
|
||||
snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
|
||||
//snprintf(puf, sizeof(puf)-1, "%s LIST \"\" %%\r\n", tag);
|
||||
snprintf(puf, sizeof(puf)-1, "%s LIST \"\" \"*\"\r\n", tag);
|
||||
|
||||
write1(data->net, puf, strlen(puf));
|
||||
|
||||
p = NULL;
|
||||
write1(sd, puf, use_ssl, data->ssl);
|
||||
|
||||
while(1){
|
||||
n = recvtimeoutssl(data->net, puf, sizeof(puf));
|
||||
if(n < 0) return ERR;
|
||||
|
||||
if(pos + n >= len){
|
||||
q = realloc(buf, len+MAXBUFSIZE+1);
|
||||
if(!q){
|
||||
printf("realloc failure: %d bytes\n", pos+MAXBUFSIZE+1);
|
||||
goto ENDE_FOLDERS;
|
||||
}
|
||||
|
||||
buf = q;
|
||||
memset(buf+pos, 0, MAXBUFSIZE+1);
|
||||
len += MAXBUFSIZE+1;
|
||||
n = recvtimeoutssl(sd, puf, sizeof(puf), 10, use_ssl, data->ssl);
|
||||
if(len + n < sizeof(buf)){
|
||||
memcpy(&buf[len], puf, n);
|
||||
len += n;
|
||||
}
|
||||
else break;
|
||||
|
||||
memcpy(buf + pos, puf, n);
|
||||
pos += n;
|
||||
|
||||
p = strstr(buf, tagok);
|
||||
if(p) break;
|
||||
if(strstr(buf, tagok)) break;
|
||||
}
|
||||
|
||||
// trim the "A3 OK LIST completed" trailer off
|
||||
if(p) *p = '\0'; //-V547
|
||||
|
||||
memset(attrs, 0, sizeof(attrs));
|
||||
|
||||
p = buf;
|
||||
|
||||
p = &buf[0];
|
||||
do {
|
||||
memset(puf, 0, sizeof(puf));
|
||||
p = split(p, '\n', puf, sizeof(puf)-1, &result);
|
||||
p = split(p, '\n', puf, sizeof(puf)-1);
|
||||
trimBuffer(puf);
|
||||
|
||||
if(strncmp(puf, "* LIST ", 7) == 0 || fldrlen){
|
||||
if(strncmp(puf, "* LIST ", 7) == 0){
|
||||
|
||||
if (fldrlen)
|
||||
q = puf;
|
||||
else
|
||||
q = strstr(puf, ") \"");
|
||||
q = strstr(puf, ") \"");
|
||||
if(q){
|
||||
if (!fldrlen) {
|
||||
*q = '\0';
|
||||
snprintf(attrs, sizeof(attrs)-1, "%s", &puf[8]);
|
||||
q += 5;
|
||||
|
||||
q += 3;
|
||||
while(*q != '"') q++;
|
||||
q++;
|
||||
if(*q == ' ') q++;
|
||||
}
|
||||
if(*q == ' ') q++;
|
||||
if(*q == '"') q++;
|
||||
|
||||
if(!fldrlen && *q == '{' && q[strlen(q)-1] == '}') {
|
||||
q++;
|
||||
fldrlen = strtol(q, NULL, 10);
|
||||
} else {
|
||||
if(q[strlen(q)-1] == '"') q[strlen(q)-1] = '\0';
|
||||
|
||||
if(fldrlen) {
|
||||
int ruflen = strlen(q) * 2;
|
||||
ruf = malloc(ruflen + 1);
|
||||
if(ruf){
|
||||
snprintf(ruf, ruflen, "%s", q);
|
||||
r = ruf;
|
||||
while(*r != '\0') {
|
||||
if(*r == '\\') {
|
||||
memmove(r + 1, r, strlen(r));
|
||||
r++;
|
||||
}
|
||||
r++;
|
||||
}
|
||||
|
||||
snprintf(folder, sizeof(folder)-1, "%s", ruf);
|
||||
|
||||
free(ruf);
|
||||
}
|
||||
else {
|
||||
printf("error: ruf = malloc()\n");
|
||||
}
|
||||
|
||||
fldrlen = 0;
|
||||
} else {
|
||||
snprintf(folder, sizeof(folder)-1, "%s", q);
|
||||
}
|
||||
|
||||
if(!strstr(attrs, "\\Noselect")){
|
||||
addnode(data->imapfolders, folder);
|
||||
}
|
||||
else if(data->quiet == 0){printf("skipping "); }
|
||||
|
||||
if(data->quiet == 0){printf("=> '%s [%s]'\n", folder, attrs); }
|
||||
|
||||
memset(attrs, 0, sizeof(attrs));
|
||||
}
|
||||
strncat(folders, "\n", foldersize-1);
|
||||
strncat(folders, q, foldersize-1);
|
||||
|
||||
}
|
||||
}
|
||||
else {
|
||||
if(strncmp(puf, tagok, strlen(tagok)) == 0) {}
|
||||
}
|
||||
|
||||
} while(p);
|
||||
|
||||
|
||||
rc = OK;
|
||||
|
||||
ENDE_FOLDERS:
|
||||
|
||||
free(buf);
|
||||
|
||||
return rc;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
257
src/import.c
257
src/import.c
@ -18,29 +18,22 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
int rc=ERR;
|
||||
int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg){
|
||||
int rc=ERR, fd;
|
||||
char *rule;
|
||||
struct stat st;
|
||||
struct parser_state state;
|
||||
struct _state state;
|
||||
struct __counters counters;
|
||||
|
||||
if(data->import->delay > 0){
|
||||
struct timespec req;
|
||||
|
||||
req.tv_sec = 0;
|
||||
req.tv_nsec = 1000000 * data->import->delay;
|
||||
|
||||
nanosleep(&req, NULL);
|
||||
}
|
||||
|
||||
init_session_data(sdata, cfg);
|
||||
|
||||
if(cfg->verbosity > 1) printf("processing: %s\n", data->import->filename);
|
||||
if(cfg->verbosity > 1) printf("processing: %s\n", filename);
|
||||
|
||||
if(strcmp(data->import->filename, "-") == 0){
|
||||
if(strcmp(filename, "-") == 0){
|
||||
|
||||
if(read_from_stdin(sdata) == ERR){
|
||||
printf("ERROR: error reading from stdin\n");
|
||||
printf("error reading from stdin\n");
|
||||
return rc;
|
||||
}
|
||||
|
||||
@ -49,189 +42,179 @@ int import_message(struct session_data *sdata, struct data *data, struct counter
|
||||
}
|
||||
else {
|
||||
|
||||
if(stat(data->import->filename, &st) != 0){
|
||||
printf("ERROR: cannot stat() %s\n", data->import->filename);
|
||||
if(stat(filename, &st) != 0){
|
||||
printf("cannot stat() %s\n", filename);
|
||||
return rc;
|
||||
}
|
||||
|
||||
if(S_ISREG(st.st_mode) == 0){
|
||||
printf("ERROR: %s is not a file\n", data->import->filename);
|
||||
printf("%s is not a file\n", filename);
|
||||
return rc;
|
||||
}
|
||||
|
||||
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", data->import->filename);
|
||||
fd = open(filename, O_RDONLY);
|
||||
if(fd == -1){
|
||||
printf("cannot open %s\n", filename);
|
||||
return rc;
|
||||
}
|
||||
close(fd);
|
||||
|
||||
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
|
||||
|
||||
sdata->tot_len = st.st_size;
|
||||
}
|
||||
|
||||
|
||||
if(sdata->tot_len < cfg->min_message_size){
|
||||
printf("%s is too short: %d bytes\n", sdata->filename, sdata->tot_len);
|
||||
return rc;
|
||||
}
|
||||
|
||||
data->import->total_size += sdata->tot_len;
|
||||
|
||||
|
||||
|
||||
sdata->sent = 0;
|
||||
sdata->delivered = 0;
|
||||
|
||||
sdata->import = 1;
|
||||
|
||||
state = parse_message(sdata, 1, data, cfg);
|
||||
post_parse(sdata, &state, cfg);
|
||||
rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
|
||||
|
||||
if(sdata->sent <= 0 && sdata->delivered > 0) sdata->sent = sdata->delivered;
|
||||
|
||||
if(sdata->sent > sdata->now) sdata->sent = sdata->now;
|
||||
if(sdata->sent == -1) sdata->sent = 0;
|
||||
|
||||
/* fat chances that you won't import emails before 1990.01.01 */
|
||||
|
||||
if(sdata->sent > 631148400) sdata->retained = sdata->sent;
|
||||
|
||||
rule = check_againt_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
|
||||
|
||||
if(rule){
|
||||
if(data->quiet == 0) printf("discarding %s by archiving policy: %s\n", data->import->filename, rule);
|
||||
printf("discarding %s by archiving policy: %s\n", filename, rule);
|
||||
rc = OK;
|
||||
}
|
||||
else {
|
||||
make_digests(sdata, cfg);
|
||||
|
||||
if(sdata->hdr_len < 10){
|
||||
printf("%s: invalid message, hdr_len: %d\n", data->import->filename, sdata->hdr_len);
|
||||
rc = ERR;
|
||||
}
|
||||
else if(data->import->after > 0 && sdata->sent < data->import->after){
|
||||
if(cfg->verbosity > 1) printf("discarding older email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->after);
|
||||
rc = ERR_DISCARDED;
|
||||
}
|
||||
else if(data->import->before > 0 && sdata->sent > data->import->before){
|
||||
if(cfg->verbosity > 1) printf("discarding newer email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->before);
|
||||
rc = ERR_DISCARDED;
|
||||
}
|
||||
else {
|
||||
// When importing emails, we should add the retention value (later) to the original sent value
|
||||
sdata->retained = sdata->sent;
|
||||
|
||||
// backup original value of data->folder
|
||||
int folder = data->folder;
|
||||
rc = process_message(sdata, &state, data, cfg);
|
||||
data->folder = folder;
|
||||
unlink(state.message_id_hash);
|
||||
}
|
||||
goto ENDE;
|
||||
}
|
||||
|
||||
make_digests(sdata, cfg);
|
||||
|
||||
rc = process_message(sdata, &state, data, cfg);
|
||||
|
||||
ENDE:
|
||||
unlink(sdata->tmpframe);
|
||||
|
||||
remove_stripped_attachments(&state);
|
||||
|
||||
if(strcmp(data->import->filename, "-") == 0) unlink(sdata->ttmpfile);
|
||||
if(strcmp(filename, "-") == 0) unlink(sdata->ttmpfile);
|
||||
|
||||
|
||||
switch(rc) {
|
||||
case OK:
|
||||
counters->c_rcvd++;
|
||||
counters->c_size += sdata->tot_len;
|
||||
counters->c_stored_size += sdata->stored_len;
|
||||
bzero(&counters, sizeof(counters));
|
||||
counters.c_size += sdata->tot_len;
|
||||
update_counters(sdata, data, &counters, cfg);
|
||||
|
||||
break;
|
||||
|
||||
case ERR_EXISTS:
|
||||
rc = OK;
|
||||
|
||||
counters->c_duplicate++;
|
||||
|
||||
if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id);
|
||||
break;
|
||||
|
||||
case ERR_DISCARDED:
|
||||
rc = OK;
|
||||
|
||||
counters->c_ignore++;
|
||||
|
||||
printf("duplicate: %s (id: %s)\n", filename, sdata->ttmpfile);
|
||||
break;
|
||||
|
||||
default:
|
||||
printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile);
|
||||
printf("failed to import: %s (id: %s)\n", filename, sdata->ttmpfile);
|
||||
break;
|
||||
}
|
||||
|
||||
if(rc != OK && data->import->failed_folder){
|
||||
char *p = strrchr(data->import->filename, '/');
|
||||
if(p)
|
||||
p++;
|
||||
else
|
||||
p = data->import->filename;
|
||||
|
||||
char newpath[SMALLBUFSIZE];
|
||||
snprintf(newpath, sizeof(newpath)-2, "%s/%s", data->import->failed_folder, p);
|
||||
|
||||
if(rename(data->import->filename, newpath))
|
||||
printf("cannot move %s to %s\n", data->import->filename, newpath);
|
||||
}
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int update_import_table(struct session_data *sdata, struct data *data) {
|
||||
int ret=ERR, status=2, started=1;
|
||||
struct sql sql;
|
||||
unsigned long get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
|
||||
unsigned long id=0;
|
||||
MYSQL_BIND bind[2];
|
||||
unsigned long len[2];
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret;
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_get_folder_id), SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) goto ENDE;
|
||||
|
||||
p_bind_init(&sql);
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
sql.sql[sql.pos] = (char *)&(started); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = foldername;
|
||||
bind[0].is_null = 0;
|
||||
len[0] = strlen(foldername); bind[0].length = &len[0];
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
|
||||
bind[1].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[1].buffer = (char *)&parent_id;
|
||||
bind[1].is_null = 0;
|
||||
bind[1].length = 0;
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int get_folder_id(struct session_data *sdata, char *foldername, int parent_id){
|
||||
int id=ERR_FOLDER;
|
||||
struct sql sql;
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) return id;
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = foldername; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&parent_id; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK){
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
|
||||
|
||||
p_store_results(&sql);
|
||||
p_fetch_results(&sql);
|
||||
p_free_results(&sql);
|
||||
if(mysql_stmt_bind_param(data->stmt_get_folder_id, bind)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
|
||||
if(mysql_stmt_execute(data->stmt_get_folder_id)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
bind[0].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[0].buffer = (char *)&id;
|
||||
bind[0].is_null = 0;
|
||||
bind[0].length = 0;
|
||||
|
||||
if(mysql_stmt_bind_result(data->stmt_get_folder_id, bind)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
if(mysql_stmt_store_result(data->stmt_get_folder_id)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
mysql_stmt_fetch(data->stmt_get_folder_id);
|
||||
|
||||
CLOSE:
|
||||
mysql_stmt_close(data->stmt_get_folder_id);
|
||||
|
||||
ENDE:
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
int add_new_folder(struct session_data *sdata, char *foldername, int parent_id){
|
||||
int id=ERR_FOLDER;
|
||||
struct sql sql;
|
||||
unsigned long add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
|
||||
unsigned long id=0;
|
||||
MYSQL_BIND bind[2];
|
||||
unsigned long len[2];
|
||||
|
||||
if(foldername == NULL) return id;
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) return id;
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_folder_table), SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) goto ENDE;
|
||||
|
||||
p_bind_init(&sql);
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
sql.sql[sql.pos] = foldername; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&parent_id; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = foldername;
|
||||
bind[0].is_null = 0;
|
||||
len[0] = strlen(foldername); bind[0].length = &len[0];
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK){
|
||||
id = p_get_insert_id(&sql);
|
||||
bind[1].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[1].buffer = (char *)&parent_id;
|
||||
bind[1].is_null = 0;
|
||||
bind[1].length = 0;
|
||||
|
||||
if(mysql_stmt_bind_param(data->stmt_insert_into_folder_table, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_FOLDER_TABLE, mysql_stmt_error(data->stmt_insert_into_folder_table));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
if(mysql_stmt_execute(data->stmt_insert_into_folder_table)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_error(&(sdata->mysql)));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
id = mysql_stmt_insert_id(data->stmt_insert_into_folder_table);
|
||||
|
||||
CLOSE:
|
||||
mysql_stmt_close(data->stmt_insert_into_folder_table);
|
||||
|
||||
ENDE:
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
|
||||
28
src/import.h
28
src/import.h
@ -1,28 +0,0 @@
|
||||
/*
|
||||
* import.h, SJ
|
||||
*/
|
||||
|
||||
#ifndef _IMPORT_H
|
||||
#define _IMPORT_H
|
||||
|
||||
|
||||
int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
int update_import_table(struct session_data *sdata, struct data *data);
|
||||
|
||||
int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg);
|
||||
int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
|
||||
int connect_to_pop3_server(struct data *data);
|
||||
void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
|
||||
int connect_to_imap_server(struct data *data);
|
||||
int list_folders(struct data *data);
|
||||
int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
void send_imap_close(struct data *data);
|
||||
|
||||
void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
|
||||
#endif /* _IMPORT_H */
|
||||
@ -1,113 +0,0 @@
|
||||
/*
|
||||
* import_imap.c
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <netdb.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <getopt.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
int i, rc=ERR, ret=OK, skipmatch;
|
||||
char port_string[8], puf[SMALLBUFSIZE];
|
||||
struct addrinfo hints, *res;
|
||||
struct node *q;
|
||||
|
||||
data->net->use_ssl = 0;
|
||||
data->import->seq = 1;
|
||||
|
||||
inithash(data->imapfolders);
|
||||
|
||||
snprintf(port_string, sizeof(port_string)-1, "%d", data->import->port);
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
if((rc = getaddrinfo(data->import->server, port_string, &hints, &res)) != 0){
|
||||
printf("getaddrinfo for '%s': %s\n", data->import->server, gai_strerror(rc));
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(data->import->port == 993) data->net->use_ssl = 1;
|
||||
|
||||
|
||||
if((data->net->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
|
||||
printf("cannot create socket\n");
|
||||
ret = ERR;
|
||||
goto ENDE_IMAP;
|
||||
}
|
||||
|
||||
if(connect(data->net->socket, res->ai_addr, res->ai_addrlen) == -1){
|
||||
printf("connect()\n");
|
||||
ret = ERR;
|
||||
goto ENDE_IMAP;
|
||||
}
|
||||
|
||||
if(connect_to_imap_server(data) == ERR){
|
||||
close(data->net->socket);
|
||||
ret = ERR;
|
||||
goto ENDE_IMAP;
|
||||
}
|
||||
|
||||
|
||||
if(list_folders(data) == ERR) goto ENDE_IMAP;
|
||||
|
||||
|
||||
for(i=0;i<MAXHASH;i++){
|
||||
q = data->imapfolders[i];
|
||||
while(q != NULL){
|
||||
|
||||
if(q->str && strlen(q->str) > 1){
|
||||
|
||||
skipmatch = 0;
|
||||
|
||||
if(data->import->skiplist && strlen(data->import->skiplist) > 0){
|
||||
snprintf(puf, sizeof(puf)-1, "%s,", (char *)q->str);
|
||||
if(strstr(data->import->skiplist, puf)) skipmatch = 1;
|
||||
}
|
||||
|
||||
if(skipmatch == 1){
|
||||
if(data->quiet == 0) printf("SKIPPING FOLDER: %s\n", (char *)q->str);
|
||||
}
|
||||
else {
|
||||
if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
|
||||
|
||||
if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
q = q->r;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
send_imap_close(data);
|
||||
|
||||
close_connection(data->net);
|
||||
|
||||
ENDE_IMAP:
|
||||
freeaddrinfo(res);
|
||||
|
||||
clearhash(data->imapfolders);
|
||||
|
||||
data->import->status = 2;
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -1,151 +0,0 @@
|
||||
/*
|
||||
* import_mailbox.c
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <netdb.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <getopt.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
FILE *F, *f=NULL;
|
||||
int rc=ERR, tot_msgs=0, ret=OK;
|
||||
char buf[MAXBUFSIZE];
|
||||
time_t t;
|
||||
|
||||
|
||||
F = fopen(mailbox, "r");
|
||||
if(!F){
|
||||
printf("cannot open mailbox: %s\n", mailbox);
|
||||
return rc;
|
||||
}
|
||||
|
||||
t = time(NULL);
|
||||
|
||||
while(fgets(buf, sizeof(buf)-1, F)){
|
||||
|
||||
if(buf[0] == 'F' && buf[1] == 'r' && buf[2] == 'o' && buf[3] == 'm' && buf[4] == ' '){
|
||||
tot_msgs++;
|
||||
if(f){
|
||||
fclose(f);
|
||||
f = NULL;
|
||||
rc = import_message(sdata, data, counters, cfg);
|
||||
if(rc == ERR){
|
||||
printf("error importing: '%s'\n", data->import->filename);
|
||||
ret = ERR;
|
||||
}
|
||||
else unlink(data->import->filename);
|
||||
|
||||
if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
|
||||
}
|
||||
|
||||
snprintf(data->import->filename, sizeof(data->import->filename)-1, "%ld-%d", t, data->import->tot_msgs);
|
||||
f = fopen(data->import->filename, "w+");
|
||||
continue;
|
||||
}
|
||||
|
||||
if(f) fprintf(f, "%s", buf);
|
||||
}
|
||||
|
||||
if(f){
|
||||
fclose(f);
|
||||
rc = import_message(sdata, data, counters, cfg);
|
||||
if(rc == ERR){
|
||||
printf("ERROR: error importing: '%s'\n", data->import->filename);
|
||||
ret = ERR;
|
||||
}
|
||||
else unlink(data->import->filename);
|
||||
|
||||
if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
|
||||
}
|
||||
|
||||
fclose(F);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
DIR *dir;
|
||||
struct dirent *de;
|
||||
int rc=ERR, ret=OK, i=0;
|
||||
int folder;
|
||||
char fname[SMALLBUFSIZE];
|
||||
struct stat st;
|
||||
|
||||
dir = opendir(directory);
|
||||
if(!dir){
|
||||
printf("cannot open directory: %s\n", directory);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
while((de = readdir(dir))){
|
||||
if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
|
||||
|
||||
snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
|
||||
|
||||
if(stat(fname, &st) == 0){
|
||||
if(S_ISDIR(st.st_mode)){
|
||||
folder = data->folder;
|
||||
rc = import_mbox_from_dir(fname, sdata, data, counters, cfg);
|
||||
data->folder = folder;
|
||||
if(rc == ERR) ret = ERR;
|
||||
}
|
||||
else {
|
||||
|
||||
if(S_ISREG(st.st_mode)){
|
||||
if(i == 0 && data->recursive_folder_names == 1){
|
||||
folder = get_folder_id(sdata, fname, data->folder);
|
||||
if(folder == ERR_FOLDER){
|
||||
folder = add_new_folder(sdata, fname, data->folder);
|
||||
|
||||
if(folder == ERR_FOLDER){
|
||||
printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
|
||||
return ERR;
|
||||
}
|
||||
else {
|
||||
data->folder = folder;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rc = import_from_mailbox(fname, sdata, data, counters, cfg);
|
||||
if(rc == OK) (data->import->tot_msgs)++;
|
||||
else ret = ERR;
|
||||
|
||||
i++;
|
||||
}
|
||||
else {
|
||||
printf("%s is not a file\n", fname);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
else {
|
||||
printf("cannot stat() %s\n", fname);
|
||||
}
|
||||
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
@ -1,111 +0,0 @@
|
||||
/*
|
||||
* import_maildir.c
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <netdb.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <getopt.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){
|
||||
DIR *dir;
|
||||
struct dirent *de;
|
||||
int rc=ERR, ret=OK, i=0;
|
||||
int folder;
|
||||
char *p, subdir[SMALLBUFSIZE];
|
||||
struct stat st;
|
||||
|
||||
dir = opendir(directory);
|
||||
if(!dir){
|
||||
printf("cannot open directory: %s\n", directory);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
while((de = readdir(dir))){
|
||||
if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
|
||||
|
||||
snprintf(data->import->filename, SMALLBUFSIZE-1, "%s/%s", directory, de->d_name);
|
||||
|
||||
if(stat(data->import->filename, &st) == 0){
|
||||
if(S_ISDIR(st.st_mode)){
|
||||
folder = data->folder;
|
||||
snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename);
|
||||
rc = import_from_maildir(sdata, data, subdir, counters, cfg);
|
||||
data->folder = folder;
|
||||
if(rc == ERR) ret = ERR;
|
||||
}
|
||||
else {
|
||||
|
||||
if(S_ISREG(st.st_mode)){
|
||||
if(i == 0 && data->recursive_folder_names == 1){
|
||||
p = strrchr(directory, '/');
|
||||
if(p) p++;
|
||||
else {
|
||||
printf("ERROR: invalid directory name: '%s'\n", directory);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
folder = get_folder_id(sdata, p, data->folder);
|
||||
if(folder == ERR_FOLDER){
|
||||
folder = add_new_folder(sdata, p, data->folder);
|
||||
|
||||
if(folder == ERR_FOLDER){
|
||||
printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
|
||||
return ERR;
|
||||
}
|
||||
else {
|
||||
data->folder = folder;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rc = import_message(sdata, data, counters, cfg);
|
||||
|
||||
if(rc == OK) (data->import->tot_msgs)++;
|
||||
else if(rc == ERR){
|
||||
printf("ERROR: error importing: '%s'\n", data->import->filename);
|
||||
ret = ERR;
|
||||
}
|
||||
|
||||
if(data->import->remove_after_import == 1 && rc != ERR) unlink(data->import->filename);
|
||||
|
||||
i++;
|
||||
|
||||
if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
|
||||
}
|
||||
else {
|
||||
printf("%s is not a file\n", data->import->filename);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
else {
|
||||
printf("cannot stat() %s\n", data->import->filename);
|
||||
}
|
||||
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
if(data->import->table_id > 0){
|
||||
update_import_table(sdata, data);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -1,114 +0,0 @@
|
||||
/*
|
||||
* import_pop3.c
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <netdb.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <getopt.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
close(data->import->fd);
|
||||
data->import->fd = -1;
|
||||
|
||||
if(import_message(sdata, data, counters, cfg) != ERR){
|
||||
unlink(data->import->filename);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
|
||||
if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
|
||||
if((*count) > 0){
|
||||
import_the_file(sdata, data, counters, cfg);
|
||||
}
|
||||
|
||||
(*count)++;
|
||||
|
||||
snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count);
|
||||
|
||||
data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
|
||||
if(data->import->fd == -1){
|
||||
// Do some error handling
|
||||
printf("error: cannot open %s\n", data->import->filename);
|
||||
}
|
||||
|
||||
} else {
|
||||
if(write(data->import->fd, buf, buflen) != buflen){
|
||||
printf("error: didnt write %d bytes\n", buflen);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
int n, rc, nullbyte, savedlen=0, puflen;
|
||||
uint64 count=0;
|
||||
char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
|
||||
|
||||
memset(savedbuf, 0, sizeof(savedbuf));
|
||||
|
||||
data->import->fd = -1;
|
||||
|
||||
do {
|
||||
memset(buf, 0, sizeof(buf));
|
||||
n = fread(buf, 1, sizeof(buf)-1, stdin);
|
||||
|
||||
int remaininglen = n;
|
||||
|
||||
if(savedlen > 0){
|
||||
memset(copybuf, 0, sizeof(copybuf));
|
||||
|
||||
memcpy(copybuf, savedbuf, savedlen);
|
||||
memcpy(©buf[savedlen], buf, n);
|
||||
|
||||
remaininglen += savedlen;
|
||||
|
||||
savedlen = 0;
|
||||
memset(savedbuf, 0, sizeof(savedbuf));
|
||||
|
||||
p = ©buf[0];
|
||||
}
|
||||
else {
|
||||
p = &buf[0];
|
||||
}
|
||||
|
||||
do {
|
||||
puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf), &rc, &nullbyte);
|
||||
p += puflen;
|
||||
remaininglen -= puflen;
|
||||
|
||||
if(puflen > 0){
|
||||
if(rc == OK){
|
||||
process_buffer(puf, puflen, &count, sdata, data, counters, cfg);
|
||||
}
|
||||
else {
|
||||
snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
|
||||
savedlen = puflen;
|
||||
}
|
||||
}
|
||||
|
||||
} while(puflen > 0);
|
||||
|
||||
} while(n > 0);
|
||||
|
||||
if(data->import->fd != -1){
|
||||
import_the_file(sdata, data, counters, cfg);
|
||||
}
|
||||
}
|
||||
@ -1,67 +0,0 @@
|
||||
/*
|
||||
* import_pop3.c
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <netdb.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <getopt.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
int rc;
|
||||
char port_string[8];
|
||||
struct addrinfo hints, *res;
|
||||
|
||||
data->net->use_ssl = 0;
|
||||
|
||||
snprintf(port_string, sizeof(port_string)-1, "%d", data->import->port);
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
if((rc = getaddrinfo(data->import->server, port_string, &hints, &res)) != 0){
|
||||
printf("getaddrinfo for '%s': %s\n", data->import->server, gai_strerror(rc));
|
||||
return;
|
||||
}
|
||||
|
||||
if(data->import->port == 995) data->net->use_ssl = 1;
|
||||
|
||||
if((data->net->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
|
||||
printf("cannot create socket\n");
|
||||
goto ENDE_POP3;
|
||||
}
|
||||
|
||||
if(connect(data->net->socket, res->ai_addr, res->ai_addrlen) == -1){
|
||||
printf("connect()\n");
|
||||
goto ENDE_POP3;
|
||||
}
|
||||
|
||||
|
||||
if(connect_to_pop3_server(data) == ERR){
|
||||
close(data->net->socket);
|
||||
goto ENDE_POP3;
|
||||
}
|
||||
|
||||
process_pop3_emails(sdata, data, counters, cfg);
|
||||
|
||||
close_connection(data->net);
|
||||
|
||||
ENDE_POP3:
|
||||
freeaddrinfo(res);
|
||||
}
|
||||
97
src/list.c
Normal file
97
src/list.c
Normal file
@ -0,0 +1,97 @@
|
||||
/*
|
||||
* list.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include "list.h"
|
||||
#include "config.h"
|
||||
|
||||
|
||||
int append_list(struct list **list, char *p){
|
||||
struct list *q, *t, *u=NULL;
|
||||
|
||||
q = *list;
|
||||
|
||||
while(q){
|
||||
if(strcmp(q->s, p) == 0)
|
||||
return 0;
|
||||
|
||||
u = q;
|
||||
q = q->r;
|
||||
}
|
||||
|
||||
t = create_list_item(p);
|
||||
if(t){
|
||||
if(*list == NULL)
|
||||
*list = t;
|
||||
else if(u)
|
||||
u->r = t;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
struct list *create_list_item(char *s){
|
||||
struct list *h=NULL;
|
||||
|
||||
if((h = malloc(sizeof(struct list))) == NULL)
|
||||
return NULL;
|
||||
|
||||
snprintf(h->s, SMALLBUFSIZE-1, "%s", s);
|
||||
h->r = NULL;
|
||||
|
||||
return h;
|
||||
}
|
||||
|
||||
|
||||
int is_string_on_list(struct list *list, char *s){
|
||||
struct list *p;
|
||||
|
||||
p = list;
|
||||
|
||||
while(p != NULL){
|
||||
if(strcmp(p->s, s) == 0) return 1;
|
||||
p = p->r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int is_item_on_string(struct list *list, char *s){
|
||||
struct list *p;
|
||||
|
||||
p = list;
|
||||
|
||||
while(p != NULL){
|
||||
if(strstr(s, p->s)) return 1;
|
||||
p = p->r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
void free_list(struct list *list){
|
||||
struct list *p, *q;
|
||||
|
||||
p = list;
|
||||
|
||||
while(p != NULL){
|
||||
q = p->r;
|
||||
|
||||
if(p)
|
||||
free(p);
|
||||
|
||||
p = q;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
17
src/list.h
Normal file
17
src/list.h
Normal file
@ -0,0 +1,17 @@
|
||||
/*
|
||||
* list.h, SJ
|
||||
*/
|
||||
|
||||
#ifndef _LIST_H
|
||||
#define _LIST_H
|
||||
|
||||
#include "defs.h"
|
||||
|
||||
int append_list(struct list **list, char *p);
|
||||
struct list *create_list_item(char *s);
|
||||
int is_string_on_list(struct list *list, char *s);
|
||||
int is_item_on_string(struct list *list, char *s);
|
||||
void free_list(struct list *list);
|
||||
|
||||
#endif /* _LIST_H */
|
||||
|
||||
89
src/memc.c
89
src/memc.c
@ -59,7 +59,7 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
|
||||
|
||||
|
||||
int set_socket_options(struct memcached_server *ptr){
|
||||
int error, flag=1, flags;
|
||||
int error, flag=1, flags, rval;
|
||||
struct timeval waittime;
|
||||
struct linger linger;
|
||||
|
||||
@ -128,7 +128,8 @@ int set_socket_options(struct memcached_server *ptr){
|
||||
|
||||
|
||||
if((flags & O_NONBLOCK) == 0){
|
||||
if(fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK) == -1) return MEMCACHED_FAILURE;
|
||||
rval = fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK);
|
||||
if(rval == -1) return MEMCACHED_FAILURE;
|
||||
}
|
||||
|
||||
return MEMCACHED_SUCCESS;
|
||||
@ -196,7 +197,7 @@ int memcached_connect(struct memcached_server *ptr){
|
||||
{
|
||||
(void)close(ptr->fd);
|
||||
ptr->fd= -1;
|
||||
|
||||
|
||||
}
|
||||
|
||||
return MEMCACHED_FAILURE;
|
||||
@ -216,17 +217,16 @@ int memcached_shutdown(struct memcached_server *ptr){
|
||||
}
|
||||
|
||||
|
||||
int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
|
||||
int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
|
||||
int len=0;
|
||||
|
||||
if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
|
||||
|
||||
// cmd could be either 'add' or 'set'
|
||||
snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %u %lu %u \r\n", cmd, key, flags, expiry, valuelen);
|
||||
snprintf(ptr->buf, MAXBUFSIZE-1, "add %s %d %ld %d \r\n", key, flags, expiry, valuelen);
|
||||
len = strlen(ptr->buf);
|
||||
|
||||
strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
|
||||
strncat(ptr->buf, "\r\n", MAXBUFSIZE-strlen(ptr->buf)-1);
|
||||
strncat(ptr->buf, value, MAXBUFSIZE-1);
|
||||
strncat(ptr->buf, "\r\n", MAXBUFSIZE-1);
|
||||
|
||||
len += valuelen + 2;
|
||||
|
||||
@ -240,7 +240,30 @@ int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *valu
|
||||
}
|
||||
|
||||
|
||||
int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result){
|
||||
int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
|
||||
int len=0;
|
||||
|
||||
if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
|
||||
|
||||
snprintf(ptr->buf, MAXBUFSIZE-1, "set %s %d %ld %d \r\n", key, flags, expiry, valuelen);
|
||||
len = strlen(ptr->buf);
|
||||
|
||||
strncat(ptr->buf, value, MAXBUFSIZE-1);
|
||||
strncat(ptr->buf, "\r\n", MAXBUFSIZE-1);
|
||||
|
||||
len += valuelen + 2;
|
||||
|
||||
send(ptr->fd, ptr->buf, len, 0);
|
||||
|
||||
ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
|
||||
|
||||
if(strcmp("STORED\r\n", ptr->buf)) return MEMCACHED_FAILURE;
|
||||
|
||||
return MEMCACHED_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result){
|
||||
char *p;
|
||||
|
||||
if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
|
||||
@ -262,6 +285,52 @@ int memcached_increment(struct memcached_server *ptr, char *key, unsigned long l
|
||||
}
|
||||
|
||||
|
||||
char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags){
|
||||
char *p;
|
||||
|
||||
if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return NULL;
|
||||
|
||||
snprintf(ptr->buf, MAXBUFSIZE, "get %s \r\n", key);
|
||||
send(ptr->fd, ptr->buf, strlen(ptr->buf), 0);
|
||||
|
||||
ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
|
||||
|
||||
if(ptr->last_read_bytes <= 0){
|
||||
memcached_shutdown(ptr);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
if(ptr->last_read_bytes < 10) return NULL;
|
||||
|
||||
if(strncmp("VALUE ", ptr->buf, 6)) return NULL;
|
||||
|
||||
p = strchr(ptr->buf, '\r');
|
||||
if(!p) return NULL;
|
||||
*p = '\0';
|
||||
ptr->result = p + 2;
|
||||
|
||||
p = strrchr(ptr->buf + 6, ' ');
|
||||
if(!p) return NULL;
|
||||
*len = atoi(p+1);
|
||||
*p = '\0';
|
||||
|
||||
|
||||
p = strrchr(ptr->buf + 6, ' ');
|
||||
if(!p) return NULL;
|
||||
*flags = atoi(p+1);
|
||||
*p = '\0';
|
||||
|
||||
|
||||
p = strchr(ptr->result, '\r');
|
||||
if(!p) return NULL;
|
||||
|
||||
*p = '\0';
|
||||
|
||||
return ptr->result;
|
||||
}
|
||||
|
||||
|
||||
int memcached_mget(struct memcached_server *ptr, char *key){
|
||||
|
||||
if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
|
||||
@ -325,3 +394,5 @@ char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *valu
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -9,8 +9,9 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
|
||||
int set_socket_options(struct memcached_server *ptr);
|
||||
int memcached_connect(struct memcached_server *ptr);
|
||||
int memcached_shutdown(struct memcached_server *ptr);
|
||||
int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
|
||||
int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result);
|
||||
int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
|
||||
int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
|
||||
int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result);
|
||||
char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags);
|
||||
int memcached_mget(struct memcached_server *ptr, char *key);
|
||||
char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *value, unsigned int *flags);
|
||||
|
||||
686
src/message.c
686
src/message.c
@ -10,7 +10,6 @@
|
||||
#include <sys/mman.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/stat.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <syslog.h>
|
||||
@ -18,154 +17,172 @@
|
||||
#include <zlib.h>
|
||||
|
||||
|
||||
int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg){
|
||||
int store_index_data(struct session_data *sdata, struct _state *state, struct __data *data, uint64 id, struct __config *cfg){
|
||||
int rc=ERR;
|
||||
char *subj, *sender=state->b_from, *sender_domain=state->b_from_domain;
|
||||
struct sql sql;
|
||||
char *subj;
|
||||
|
||||
if(data->folder == 0){
|
||||
data->folder = get_folder_id_by_rule(data, state, sdata->tot_len, sdata->spam_message, cfg);
|
||||
}
|
||||
MYSQL_BIND bind[14];
|
||||
unsigned long len[14];
|
||||
|
||||
subj = state->b_subject;
|
||||
if(*subj == ' ') subj++;
|
||||
|
||||
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_sphinx_table), SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
|
||||
|
||||
|
||||
fix_email_address_for_sphinx(state->b_from);
|
||||
fix_email_address_for_sphinx(state->b_sender);
|
||||
fix_email_address_for_sphinx(state->b_to);
|
||||
fix_email_address_for_sphinx(state->b_from_domain);
|
||||
fix_email_address_for_sphinx(state->b_sender_domain);
|
||||
fix_email_address_for_sphinx(state->b_to_domain);
|
||||
|
||||
if(state->b_sender_domain[0]){
|
||||
sender = state->b_sender;
|
||||
sender_domain = state->b_sender_domain;
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
bind[0].buffer = (char *)&id;
|
||||
bind[0].is_null = 0;
|
||||
bind[0].length = 0;
|
||||
|
||||
bind[1].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[1].buffer = state->b_from;
|
||||
bind[1].is_null = 0;
|
||||
len[1] = strlen(state->b_from); bind[1].length = &len[1];
|
||||
|
||||
bind[2].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[2].buffer = state->b_to;
|
||||
bind[2].is_null = 0;
|
||||
len[2] = strlen(state->b_to); bind[2].length = &len[2];
|
||||
|
||||
bind[3].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[3].buffer = state->b_from_domain;
|
||||
bind[3].is_null = 0;
|
||||
len[3] = strlen(state->b_from_domain); bind[3].length = &len[3];
|
||||
|
||||
bind[4].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[4].buffer = state->b_to_domain;
|
||||
bind[4].is_null = 0;
|
||||
len[4] = strlen(state->b_to_domain); bind[4].length = &len[4];
|
||||
|
||||
bind[5].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[5].buffer = subj;
|
||||
bind[5].is_null = 0;
|
||||
len[5] = strlen(subj); bind[5].length = &len[5];
|
||||
|
||||
bind[6].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[6].buffer = state->b_body;
|
||||
bind[6].is_null = 0;
|
||||
len[6] = strlen(state->b_body); bind[6].length = &len[6];
|
||||
|
||||
bind[7].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[7].buffer = (char *)&sdata->now;
|
||||
bind[7].is_null = 0;
|
||||
bind[7].length = 0;
|
||||
|
||||
bind[8].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[8].buffer = (char *)&sdata->sent;
|
||||
bind[8].is_null = 0;
|
||||
bind[8].length = 0;
|
||||
|
||||
bind[9].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[9].buffer = (char *)&sdata->tot_len;
|
||||
bind[9].is_null = 0;
|
||||
bind[9].length = 0;
|
||||
|
||||
bind[10].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[10].buffer = (char *)&sdata->direction;
|
||||
bind[10].is_null = 0;
|
||||
bind[10].length = 0;
|
||||
|
||||
bind[11].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[11].buffer = (char *)&data->folder;
|
||||
bind[11].is_null = 0;
|
||||
bind[11].length = 0;
|
||||
|
||||
bind[12].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[12].buffer = (char *)&state->n_attachments;
|
||||
bind[12].is_null = 0;
|
||||
bind[12].length = 0;
|
||||
|
||||
bind[13].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[13].buffer = sdata->attachments;
|
||||
bind[13].is_null = 0;
|
||||
len[13] = strlen(sdata->attachments); bind[13].length = &len[13];
|
||||
|
||||
|
||||
if(mysql_stmt_bind_param(data->stmt_insert_into_sphinx_table, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(data->stmt_insert_into_sphinx_table));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
if(cfg->rtindex){
|
||||
// Manticore doesn't support prepared statements using sphinxQL
|
||||
// so we have to go through a painful query assembly escaping
|
||||
// the untrusted input
|
||||
//
|
||||
char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
|
||||
char *query=NULL;
|
||||
|
||||
snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
|
||||
|
||||
int ret = append_string_to_buffer(&query, a);
|
||||
|
||||
unsigned long len = strlen(sender);
|
||||
char *s = malloc(2*len+1);
|
||||
mysql_real_escape_string(&(sdata->sphx), s, sender, len);
|
||||
ret += append_string_to_buffer(&query, s);
|
||||
free(s);
|
||||
ret += append_string_to_buffer(&query, "','");
|
||||
|
||||
len = strlen(state->b_to);
|
||||
s = malloc(2*len+1);
|
||||
mysql_real_escape_string(&(sdata->sphx), s, state->b_to, len);
|
||||
ret += append_string_to_buffer(&query, s);
|
||||
free(s);
|
||||
ret += append_string_to_buffer(&query, "','");
|
||||
|
||||
len = strlen(sender_domain);
|
||||
s = malloc(2*len+1);
|
||||
mysql_real_escape_string(&(sdata->sphx), s, sender_domain, len);
|
||||
ret += append_string_to_buffer(&query, s);
|
||||
free(s);
|
||||
ret += append_string_to_buffer(&query, "','");
|
||||
|
||||
len = strlen(state->b_to_domain);
|
||||
s = malloc(2*len+1);
|
||||
mysql_real_escape_string(&(sdata->sphx), s, state->b_to_domain, len);
|
||||
ret += append_string_to_buffer(&query, s);
|
||||
free(s);
|
||||
ret += append_string_to_buffer(&query, "','");
|
||||
|
||||
len = strlen(subj);
|
||||
s = malloc(2*len+1);
|
||||
mysql_real_escape_string(&(sdata->sphx), s, subj, len);
|
||||
ret += append_string_to_buffer(&query, s);
|
||||
free(s);
|
||||
ret += append_string_to_buffer(&query, "','");
|
||||
|
||||
len = strlen(state->b_body);
|
||||
s = malloc(2*len+1);
|
||||
mysql_real_escape_string(&(sdata->sphx), s, state->b_body, len);
|
||||
ret += append_string_to_buffer(&query, s);
|
||||
free(s);
|
||||
ret += append_string_to_buffer(&query, "')");
|
||||
|
||||
if(ret == 0 && mysql_real_query(&(sdata->sphx), query, strlen(query)) == OK) rc = OK;
|
||||
else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, errno=%d, append ret=%d", sdata->ttmpfile, id, mysql_errno(&(sdata->sphx)), ret);
|
||||
|
||||
free(query);
|
||||
if(mysql_stmt_execute(data->stmt_insert_into_sphinx_table)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_error(&(sdata->mysql)));
|
||||
goto CLOSE;
|
||||
}
|
||||
else {
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
|
||||
|
||||
p_bind_init(&sql);
|
||||
rc = OK;
|
||||
|
||||
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
|
||||
sql.sql[sql.pos] = sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = sdata->attachments; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
CLOSE:
|
||||
mysql_stmt_close(data->stmt_insert_into_sphinx_table);
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
|
||||
else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, sql_errno=%d", sdata->ttmpfile, id, sdata->sql_errno);
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
uint64 get_metaid_by_messageid(struct session_data *sdata, char *message_id, char *piler_id){
|
||||
uint64 get_metaid_by_messageid(struct session_data *sdata, struct __data *data, char *message_id, struct __config *cfg){
|
||||
unsigned long len=0;
|
||||
uint64 id=0;
|
||||
struct sql sql;
|
||||
MYSQL_BIND bind[1];
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID) == ERR) return id;
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
p_bind_init(&sql);
|
||||
sql.sql[sql.pos] = message_id; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = message_id;
|
||||
bind[0].is_null = 0;
|
||||
len = strlen(message_id); bind[0].length = &len;
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK){
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
sql.sql[sql.pos] = piler_id; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
|
||||
|
||||
p_store_results(&sql);
|
||||
|
||||
p_fetch_results(&sql);
|
||||
|
||||
p_free_results(&sql);
|
||||
if(mysql_stmt_bind_param(data->stmt_get_meta_id_by_message_id, bind)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
if(mysql_stmt_execute(data->stmt_get_meta_id_by_message_id)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
bind[0].buffer = (char *)&id;
|
||||
bind[0].is_null = 0;
|
||||
bind[0].length = 0;
|
||||
|
||||
|
||||
if(mysql_stmt_bind_result(data->stmt_get_meta_id_by_message_id, bind)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
if(mysql_stmt_store_result(data->stmt_get_meta_id_by_message_id)){
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
mysql_stmt_fetch(data->stmt_get_meta_id_by_message_id);
|
||||
|
||||
CLOSE:
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
int store_recipients(struct session_data *sdata, char *to, uint64 id, struct config *cfg){
|
||||
int rc=OK, n=0;
|
||||
int store_recipients(struct session_data *sdata, struct __data *data, char *to, uint64 id, int log_errors, struct __config *cfg){
|
||||
int ret=OK, n=0;
|
||||
char *p, *q, puf[SMALLBUFSIZE];
|
||||
struct sql sql;
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE) == ERR) return ERR;
|
||||
MYSQL_BIND bind[3];
|
||||
unsigned long len[3];
|
||||
|
||||
|
||||
p = to;
|
||||
do {
|
||||
@ -176,289 +193,306 @@ int store_recipients(struct session_data *sdata, char *to, uint64 id, struct con
|
||||
if(q && strlen(q) > 3 && does_it_seem_like_an_email_address(puf) == 1){
|
||||
q++;
|
||||
|
||||
p_bind_init(&sql);
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
|
||||
sql.sql[sql.pos] = &puf[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = q; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
bind[0].buffer = (char *)&id;
|
||||
bind[0].is_null = 0;
|
||||
bind[0].length = 0;
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == ERR){
|
||||
if(sdata->sql_errno != ER_DUP_ENTRY){
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: failed to add '%s' for id=%llu to rcpt table, sql_errno=%d", sdata->ttmpfile, puf, id, sdata->sql_errno);
|
||||
rc = ERR;
|
||||
}
|
||||
bind[1].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[1].buffer = &puf[0];
|
||||
bind[1].is_null = 0;
|
||||
len[1] = strlen(puf); bind[1].length = &len[1];
|
||||
|
||||
bind[2].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[2].buffer = q;
|
||||
bind[2].is_null = 0;
|
||||
len[2] = strlen(q); bind[2].length = &len[2];
|
||||
|
||||
if(mysql_stmt_bind_param(data->stmt_insert_into_rcpt_table, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(data->stmt_insert_into_rcpt_table));
|
||||
ret = ERR;
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
if(mysql_stmt_execute(data->stmt_insert_into_rcpt_table) && log_errors == 1){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_error(&(sdata->mysql)));
|
||||
ret = ERR;
|
||||
}
|
||||
else n++;
|
||||
}
|
||||
|
||||
} while(p);
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored %d recipients, rc=%d", sdata->ttmpfile, n, rc);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int store_folder_id(struct session_data *sdata, struct data *data, uint64 id, struct config *cfg){
|
||||
int rc=ERR;
|
||||
struct sql sql;
|
||||
|
||||
if(data->folder == ERR_FOLDER) return rc;
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE) == ERR) return rc;
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
|
||||
close_prepared_statement(&sql);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored folderdata, rc=%d", sdata->ttmpfile, rc);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int update_metadata_reference(struct session_data *sdata, struct parser_state *state, char *ref, struct config *cfg){
|
||||
int ret = ERR;
|
||||
struct sql sql;
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE) == ERR) return ret;
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = ref; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = state->reference; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated meta reference for '%s', rc=%d", sdata->ttmpfile, state->reference, ret);
|
||||
CLOSE:
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: added %d recipients", sdata->ttmpfile, n);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int store_meta_data(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
|
||||
int rc=ERR;
|
||||
char *subj, *sender, *sender_domain, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
|
||||
uint64 id=0;
|
||||
struct sql sql;
|
||||
int update_metadata_reference(struct session_data *sdata, struct _state *state, struct __data *data, char *ref, struct __config *cfg){
|
||||
int rc, ret = ERR;
|
||||
MYSQL_BIND bind[2];
|
||||
unsigned long len[2];
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = ref;
|
||||
bind[0].is_null = 0;
|
||||
len[0] = strlen(ref); bind[0].length = &len[0];
|
||||
|
||||
bind[1].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[1].buffer = state->reference;
|
||||
bind[1].is_null = 0;
|
||||
len[1] = strlen(state->reference); bind[1].length = &len[1];
|
||||
|
||||
if(mysql_stmt_bind_param(data->stmt_update_metadata_reference, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(data->stmt_update_metadata_reference));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
rc = mysql_stmt_execute(data->stmt_update_metadata_reference);
|
||||
if(rc == 0) ret = OK;
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated meta reference for '%s', rc=%d", sdata->ttmpfile, state->reference, rc);
|
||||
|
||||
CLOSE:
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int store_meta_data(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
|
||||
int rc, ret=ERR;
|
||||
char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
|
||||
|
||||
MYSQL_BIND bind[17];
|
||||
unsigned long len[17];
|
||||
|
||||
my_ulonglong id=0;
|
||||
|
||||
|
||||
subj = state->b_subject;
|
||||
if(*subj == ' ') subj++;
|
||||
|
||||
if(state->b_sender_domain[0]){
|
||||
sender = state->b_sender;
|
||||
sender_domain = state->b_sender_domain;
|
||||
get_first_email_address_from_string(state->b_sender, s2, sizeof(s2));
|
||||
} else {
|
||||
sender = state->b_from;
|
||||
sender_domain = state->b_from_domain;
|
||||
get_first_email_address_from_string(state->b_from, s2, sizeof(s2));
|
||||
}
|
||||
snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
|
||||
|
||||
snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
|
||||
|
||||
digest_string("sha256", s, &vcode[0]);
|
||||
digest_string(s, &vcode[0]);
|
||||
|
||||
memset(ref, 0, sizeof(ref));
|
||||
if(strlen(state->reference) > 10){
|
||||
digest_string("sha256", state->reference, &ref[0]);
|
||||
update_metadata_reference(sdata, state, &ref[0], cfg);
|
||||
}
|
||||
else if(state->reference[0] == 0){
|
||||
// during import, the order of messages is often random
|
||||
// check if this is a message which is already referenced
|
||||
uint64 count=0;
|
||||
|
||||
digest_string("sha256", state->message_id, &ref[0]);
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_METADATA_REFERENCE) != ERR){
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK){
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = (char *)&count; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
p_store_results(&sql);
|
||||
p_fetch_results(&sql);
|
||||
p_free_results(&sql);
|
||||
}
|
||||
}
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
|
||||
// no reference yet
|
||||
if(count <= 0)
|
||||
ref[0] = 0;
|
||||
digest_string(state->reference, &ref[0]);
|
||||
update_metadata_reference(sdata, state, data, &ref[0], cfg);
|
||||
}
|
||||
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_meta_table), SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
|
||||
|
||||
memset(s2, 0, sizeof(s2));
|
||||
|
||||
p = state->b_from;
|
||||
do {
|
||||
memset(s2, 0, sizeof(s2));
|
||||
p = split(p, ' ', s2, sizeof(s2)-1);
|
||||
|
||||
if(s2[0] == '\0') continue;
|
||||
|
||||
if(does_it_seem_like_an_email_address(s2) == 1){ break; }
|
||||
} while(p);
|
||||
|
||||
|
||||
if(strlen(state->b_to) < 5){
|
||||
snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients@no.domain");
|
||||
}
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
p_bind_init(&sql);
|
||||
bind[0].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[0].buffer = &s2[0];
|
||||
bind[0].is_null = 0;
|
||||
len[0] = strlen(s2); bind[0].length = &len[0];
|
||||
|
||||
sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->retained; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->hdr_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
|
||||
sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = state->message_id; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = sdata->digest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = sdata->bodydigest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
sql.sql[sql.pos] = &vcode[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
|
||||
bind[1].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[1].buffer = state->b_from_domain;
|
||||
bind[1].is_null = 0;
|
||||
len[1] = strlen(state->b_from_domain); bind[1].length = &len[1];
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK){
|
||||
id = p_get_insert_id(&sql);
|
||||
bind[2].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[2].buffer = subj;
|
||||
bind[2].is_null = 0;
|
||||
len[2] = strlen(subj); bind[2].length = &len[2];
|
||||
|
||||
if(store_recipients(sdata, state->b_to, id, cfg) == OK){
|
||||
bind[3].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[3].buffer = (char *)&sdata->spam_message;
|
||||
bind[3].is_null = 0;
|
||||
bind[3].length = 0;
|
||||
|
||||
if(store_index_data(sdata, state, data, id, cfg) == OK) rc = OK;
|
||||
bind[4].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[4].buffer = (char *)&sdata->now;
|
||||
bind[4].is_null = 0;
|
||||
bind[4].length = 0;
|
||||
|
||||
if(cfg->enable_folders == 1){
|
||||
rc = store_folder_id(sdata, data, id, cfg);
|
||||
}
|
||||
}
|
||||
bind[5].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[5].buffer = (char *)&sdata->sent;
|
||||
bind[5].is_null = 0;
|
||||
bind[5].length = 0;
|
||||
|
||||
// There were some sql errors, so we should rollback everything
|
||||
if(rc == ERR){
|
||||
rollback(sdata, state, id, cfg);
|
||||
}
|
||||
bind[6].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[6].buffer = (char *)&sdata->retained;
|
||||
bind[6].is_null = 0;
|
||||
bind[6].length = 0;
|
||||
|
||||
bind[7].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[7].buffer = (char *)&sdata->tot_len;
|
||||
bind[7].is_null = 0;
|
||||
bind[7].length = 0;
|
||||
|
||||
bind[8].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[8].buffer = (char *)&sdata->hdr_len;
|
||||
bind[8].is_null = 0;
|
||||
bind[8].length = 0;
|
||||
|
||||
bind[9].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[9].buffer = (char *)&sdata->direction;
|
||||
bind[9].is_null = 0;
|
||||
bind[9].length = 0;
|
||||
|
||||
bind[10].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[10].buffer = (char *)&state->n_attachments;
|
||||
bind[10].is_null = 0;
|
||||
bind[10].length = 0;
|
||||
|
||||
bind[11].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[11].buffer = sdata->ttmpfile;
|
||||
bind[11].is_null = 0;
|
||||
len[11] = strlen(sdata->ttmpfile); bind[11].length = &len[11];
|
||||
|
||||
bind[12].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[12].buffer = state->message_id;
|
||||
bind[12].is_null = 0;
|
||||
len[12] = strlen(state->message_id); bind[12].length = &len[12];
|
||||
|
||||
bind[13].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[13].buffer = &ref[0];
|
||||
bind[13].is_null = 0;
|
||||
len[13] = strlen(ref); bind[13].length = &len[13];
|
||||
|
||||
bind[14].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[14].buffer = sdata->digest;
|
||||
bind[14].is_null = 0;
|
||||
len[14] = strlen(sdata->digest); bind[14].length = &len[14];
|
||||
|
||||
bind[15].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[15].buffer = sdata->bodydigest;
|
||||
bind[15].is_null = 0;
|
||||
len[15] = strlen(sdata->bodydigest); bind[15].length = &len[15];
|
||||
|
||||
bind[16].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[16].buffer = &vcode[0];
|
||||
bind[16].is_null = 0;
|
||||
len[16] = strlen(vcode); bind[16].length = &len[16];
|
||||
|
||||
|
||||
|
||||
if(mysql_stmt_bind_param(data->stmt_insert_into_meta_table, bind)){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(data->stmt_insert_into_meta_table));
|
||||
goto CLOSE;
|
||||
}
|
||||
|
||||
|
||||
rc = mysql_stmt_execute(data->stmt_insert_into_meta_table);
|
||||
|
||||
if(rc){
|
||||
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute() error: *%s*", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_error(&(sdata->mysql)));
|
||||
ret = ERR_EXISTS;
|
||||
}
|
||||
else {
|
||||
syslog(LOG_PRIORITY, "ERROR: %s storing metadata, sql_errno=%d", sdata->ttmpfile, sdata->sql_errno);
|
||||
id = mysql_stmt_insert_id(data->stmt_insert_into_meta_table);
|
||||
|
||||
rc = store_recipients(sdata, data, state->b_to, id, 1, cfg);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc);
|
||||
|
||||
if(rc == OK){
|
||||
|
||||
rc = store_index_data(sdata, state, data, id, cfg);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
|
||||
|
||||
if(rc == OK)
|
||||
ret = OK;
|
||||
}
|
||||
}
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
CLOSE:
|
||||
mysql_stmt_close(data->stmt_insert_into_meta_table);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d", sdata->ttmpfile, rc);
|
||||
|
||||
return rc;
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
void rollback(struct session_data *sdata, struct parser_state *state, uint64 id, struct config *cfg){
|
||||
char buf[SMALLBUFSIZE];
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_SPHINX_TABLE, id);
|
||||
p_query(sdata, buf);
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_RECIPIENT_TABLE, id);
|
||||
p_query(sdata, buf);
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_METADATA_TABLE, id);
|
||||
p_query(sdata, buf);
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE piler_id='%s'", SQL_ATTACHMENT_TABLE, sdata->ttmpfile);
|
||||
p_query(sdata, buf);
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
|
||||
|
||||
if(cfg->enable_folders == 1){
|
||||
snprintf(buf, sizeof(buf)-1, "DELETE FROM " SQL_FOLDER_MESSAGE_TABLE " WHERE id=%llu", id);
|
||||
p_query(sdata, buf);
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
|
||||
}
|
||||
|
||||
remove_stored_message_files(sdata, state, cfg);
|
||||
}
|
||||
|
||||
|
||||
void remove_stripped_attachments(struct parser_state *state){
|
||||
void remove_stripped_attachments(struct _state *state){
|
||||
int i;
|
||||
|
||||
for(i=1; i<=state->n_attachments; i++) unlink(state->attachments[i].internalname);
|
||||
}
|
||||
|
||||
|
||||
int is_duplicated_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
|
||||
int fd;
|
||||
char piler_id[SMALLBUFSIZE];
|
||||
int process_message(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
|
||||
int i, rc;
|
||||
uint64 id=0;
|
||||
|
||||
/* discard if existing message_id */
|
||||
|
||||
sdata->duplicate_id = get_metaid_by_messageid(sdata, state->message_id, piler_id);
|
||||
id = get_metaid_by_messageid(sdata, data, state->message_id, cfg);
|
||||
|
||||
if(sdata->duplicate_id > 0){
|
||||
if(id > 0){
|
||||
remove_stripped_attachments(state);
|
||||
|
||||
if(sdata->restored_copy == 0 && strlen(state->b_journal_to) > 0){
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, sdata->duplicate_id, state->message_id);
|
||||
store_recipients(sdata, state->b_journal_to, sdata->duplicate_id, cfg);
|
||||
if(strlen(state->b_journal_to) > 0){
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, id, state->message_id);
|
||||
store_recipients(sdata, data, state->b_journal_to, id, 0, cfg);
|
||||
}
|
||||
|
||||
return ERR_EXISTS;
|
||||
}
|
||||
|
||||
fd = open(state->message_id_hash, O_CREAT|O_EXCL, S_IRUSR|S_IWUSR);
|
||||
if(fd == -1){
|
||||
remove_stripped_attachments(state);
|
||||
syslog(LOG_PRIORITY, "%s: touch %s FAILED (%s), error: %s", sdata->ttmpfile, state->message_id_hash, state->message_id, strerror(errno));
|
||||
return ERR_EXISTS;
|
||||
}
|
||||
close(fd);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
|
||||
/* store base64 encoded file attachments */
|
||||
|
||||
if(state->n_attachments > 0){
|
||||
rc = store_attachments(sdata, state, data, cfg);
|
||||
|
||||
if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){
|
||||
|
||||
if(strstr(data->dedup, state->message_id_hash)){
|
||||
if(cfg->verbosity >= _LOG_DEBUG){
|
||||
syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
|
||||
syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);
|
||||
}
|
||||
|
||||
remove_stripped_attachments(state);
|
||||
return ERR_EXISTS;
|
||||
for(i=1; i<=state->n_attachments; i++){
|
||||
unlink(state->attachments[i].internalname);
|
||||
}
|
||||
|
||||
memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2);
|
||||
if(rc) return ERR;
|
||||
}
|
||||
|
||||
|
||||
rc = store_file(sdata, sdata->tmpframe, 0, 0, cfg);
|
||||
if(rc == 0){
|
||||
syslog(LOG_PRIORITY, "%s: error storing message: %s", sdata->ttmpfile, sdata->tmpframe);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
sdata->retained += query_retain_period(data->retention_rules, state, sdata->tot_len, sdata->spam_message, cfg);
|
||||
|
||||
rc = store_meta_data(sdata, state, data, cfg);
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d", sdata->ttmpfile, rc);
|
||||
if(rc == ERR_EXISTS){
|
||||
|
||||
remove_stored_message_files(sdata, state, cfg);
|
||||
|
||||
return ERR_EXISTS;
|
||||
}
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
|
||||
|
||||
if(is_duplicated_message(sdata, state, data, cfg) == ERR_EXISTS)
|
||||
return ERR_EXISTS;
|
||||
|
||||
sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
|
||||
|
||||
|
||||
if(state->n_attachments > 0 && store_attachments(sdata, state, cfg) == ERR) return ERR;
|
||||
|
||||
|
||||
if(store_file(sdata, sdata->tmpframe, 0, cfg) == 0){
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: failed to store message: %s", sdata->ttmpfile, sdata->tmpframe);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
return store_meta_data(sdata, state, data, cfg);
|
||||
}
|
||||
|
||||
516
src/misc.c
516
src/misc.c
@ -19,7 +19,6 @@
|
||||
#include <unistd.h>
|
||||
#include <ctype.h>
|
||||
#include <openssl/ssl.h>
|
||||
#include <openssl/err.h>
|
||||
#include "misc.h"
|
||||
#include "smtpcodes.h"
|
||||
#include "errmsg.h"
|
||||
@ -32,52 +31,13 @@ int get_build(){
|
||||
}
|
||||
|
||||
|
||||
void get_extractor_list(){
|
||||
printf("Extractors: ");
|
||||
|
||||
#ifdef HAVE_PDFTOTEXT
|
||||
printf("%s ", HAVE_PDFTOTEXT);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_CATDOC
|
||||
printf("%s ", HAVE_CATDOC);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_CATPPT
|
||||
printf("%s ", HAVE_CATPPT);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_XLS2CSV
|
||||
printf("%s ", HAVE_XLS2CSV);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_PPTHTML
|
||||
printf("%s ", HAVE_PPTHTML);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_UNRTF
|
||||
printf("%s ", HAVE_UNRTF);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_TNEF
|
||||
printf("%s ", HAVE_TNEF);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_ZIP
|
||||
printf("libzip ");
|
||||
#endif
|
||||
|
||||
printf("\n\n");
|
||||
}
|
||||
|
||||
|
||||
void __fatal(char *s){
|
||||
fprintf(stderr, "%s\n", s);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/*
|
||||
* calculate the difference between two timevals in [usec]
|
||||
* calculate the difference betwwen two timevals in [usec]
|
||||
*/
|
||||
|
||||
long tvdiff(struct timeval a, struct timeval b){
|
||||
@ -88,31 +48,87 @@ long tvdiff(struct timeval a, struct timeval b){
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* search something in a buffer
|
||||
*/
|
||||
|
||||
int searchStringInBuffer(char *s, int len1, char *what, int len2){
|
||||
int i, k, r;
|
||||
|
||||
for(i=0; i<len1; i++){
|
||||
r = 0;
|
||||
|
||||
for(k=0; k<len2; k++){
|
||||
if(*(s+i+k) == *(what+k))
|
||||
r++;
|
||||
}
|
||||
|
||||
if(r == len2)
|
||||
return i;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* count a character in buffer
|
||||
*/
|
||||
|
||||
int countCharacterInBuffer(char *p, char c){
|
||||
int i=0;
|
||||
|
||||
for(; *p; p++){
|
||||
if(*p == c)
|
||||
i++;
|
||||
}
|
||||
|
||||
return i;
|
||||
}
|
||||
|
||||
|
||||
void replaceCharacterInBuffer(char *p, char from, char to){
|
||||
int i, k=0;
|
||||
|
||||
for(i=0; i<strlen(p); i++){
|
||||
if(p[i] == from){
|
||||
if(to > 0){
|
||||
p[k] = to;
|
||||
k++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
p[k] = p[i];
|
||||
k++;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
p[k] = '\0';
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* split a string by a character as delimiter
|
||||
*/
|
||||
|
||||
char *split(char *str, int ch, char *buf, int buflen, int *result){
|
||||
char *p;
|
||||
char *split(char *row, int ch, char *s, int size){
|
||||
char *r;
|
||||
|
||||
*result = 0;
|
||||
if(row == NULL || s == NULL)
|
||||
return NULL;
|
||||
|
||||
if(str == NULL || buf == NULL || buflen < 2) return NULL;
|
||||
r = strchr(row, ch);
|
||||
if(r) *r = '\0';
|
||||
|
||||
p = strchr(str, ch);
|
||||
if(p){
|
||||
*p = '\0';
|
||||
snprintf(s, size, "%s", row);
|
||||
|
||||
if(r){
|
||||
*r = ch;
|
||||
r++;
|
||||
}
|
||||
|
||||
snprintf(buf, buflen, "%s", str);
|
||||
|
||||
if(p){
|
||||
*p = ch;
|
||||
*result = 1;
|
||||
p++;
|
||||
}
|
||||
|
||||
return p;
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
@ -143,8 +159,10 @@ char *split_str(char *row, char *what, char *s, int size){
|
||||
r += strlen(what);
|
||||
}
|
||||
|
||||
strncpy(s, row, len);
|
||||
s[len] = '\0';
|
||||
if(s != NULL){
|
||||
strncpy(s, row, len);
|
||||
s[len] = '\0';
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
@ -158,13 +176,13 @@ int trimBuffer(char *s){
|
||||
int n=0;
|
||||
char *p;
|
||||
|
||||
p = strchr(s, '\n');
|
||||
p = strrchr(s, '\n');
|
||||
if(p){
|
||||
*p = '\0';
|
||||
n++;
|
||||
}
|
||||
|
||||
p = strchr(s, '\r');
|
||||
p = strrchr(s, '\r');
|
||||
if(p){
|
||||
*p = '\0';
|
||||
n++;
|
||||
@ -174,36 +192,6 @@ int trimBuffer(char *s){
|
||||
}
|
||||
|
||||
|
||||
int extract_verp_address(char *email){
|
||||
char *p1;
|
||||
|
||||
// a VERP address is like archive+user=domain.com@myarchive.local
|
||||
|
||||
if(!email) return 0;
|
||||
|
||||
if(strlen(email) < 5) return 0;
|
||||
|
||||
p1 = strchr(email, '+');
|
||||
if(p1){
|
||||
char *p2 = strchr(p1, '@');
|
||||
if(p2 && p2 > p1 + 2){
|
||||
if(strchr(p1+1, '=')){
|
||||
char puf[SMALLBUFSIZE];
|
||||
memset(puf, 0, sizeof(puf));
|
||||
|
||||
memcpy(&puf[0], p1+1, p2-p1-1);
|
||||
char *p = strchr(puf, '=');
|
||||
if(p) *p = '@';
|
||||
strcpy(email, puf);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int extractEmail(char *rawmail, char *email){
|
||||
char *p;
|
||||
|
||||
@ -215,7 +203,6 @@ int extractEmail(char *rawmail, char *email){
|
||||
p = strchr(email, '>');
|
||||
if(p){
|
||||
*p = '\0';
|
||||
extract_verp_address(email);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -224,36 +211,6 @@ int extractEmail(char *rawmail, char *email){
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Generate a random string from /dev/urandom or
|
||||
* using the rand() function if not possible
|
||||
*/
|
||||
|
||||
void make_random_string(unsigned char *buf, int buflen){
|
||||
const char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
|
||||
int i, fd, urandom=0, len = sizeof(alphanum)-1;
|
||||
|
||||
fd = open(RANDOM_POOL, O_RDONLY);
|
||||
if(fd != -1){
|
||||
if(readFromEntropyPool(fd, buf, buflen) == buflen){
|
||||
for(i=0; i<buflen; i++){
|
||||
*(buf+i) = alphanum[*(buf+i) % len];
|
||||
}
|
||||
|
||||
urandom = 1;
|
||||
}
|
||||
close(fd);
|
||||
}
|
||||
|
||||
if(urandom == 1) return;
|
||||
|
||||
for(i=0; i<buflen; i++){
|
||||
*(buf+i) = alphanum[rand() % len];
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
void create_id(char *id, unsigned char server_id){
|
||||
int i;
|
||||
unsigned char buf[RND_STR_LEN/2];
|
||||
@ -262,13 +219,11 @@ void create_id(char *id, unsigned char server_id){
|
||||
|
||||
get_random_bytes(buf, RND_STR_LEN/2, server_id);
|
||||
|
||||
// New encryption scheme using AES-256
|
||||
buf[0] = 0x50;
|
||||
|
||||
for(i=0; i < RND_STR_LEN/2; i++){
|
||||
sprintf(id, "%02x", buf[i]);
|
||||
id += 2;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -286,7 +241,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
|
||||
taia_now(&now);
|
||||
taia_pack(nowpack, &now);
|
||||
|
||||
memcpy(buf, nowpack, 12); //-V512
|
||||
memcpy(buf, nowpack, 12);
|
||||
|
||||
fd = open(RANDOM_POOL, O_RDONLY);
|
||||
if(fd == -1) return ret;
|
||||
@ -296,7 +251,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
|
||||
if(readFromEntropyPool(fd, buf+12+1, len-12-1) != len-12-1){
|
||||
syslog(LOG_PRIORITY, "%s: %s", ERR_CANNOT_READ_FROM_POOL, RANDOM_POOL);
|
||||
}
|
||||
|
||||
|
||||
close(fd);
|
||||
return ret;
|
||||
}
|
||||
@ -306,12 +261,12 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
|
||||
* read random data from entropy pool
|
||||
*/
|
||||
|
||||
int readFromEntropyPool(int fd, void *_s, ssize_t n){
|
||||
int readFromEntropyPool(int fd, void *_s, size_t n){
|
||||
char *s = _s;
|
||||
ssize_t pos = 0;
|
||||
ssize_t res, pos = 0;
|
||||
|
||||
while(n > pos){
|
||||
ssize_t res = read(fd, s + pos, n - pos);
|
||||
res = read(fd, s + pos, n - pos);
|
||||
switch(res){
|
||||
case -1: continue;
|
||||
case 0: return res;
|
||||
@ -347,13 +302,13 @@ int recvtimeout(int s, char *buf, int len, int timeout){
|
||||
}
|
||||
|
||||
|
||||
int write1(struct net *net, void *buf, int buflen){
|
||||
int write1(int sd, char *buf, int use_ssl, SSL *ssl){
|
||||
int n;
|
||||
|
||||
if(net->use_ssl == 1)
|
||||
n = SSL_write(net->ssl, buf, buflen);
|
||||
if(use_ssl == 1)
|
||||
n = SSL_write(ssl, buf, strlen(buf));
|
||||
else
|
||||
n = send(net->socket, buf, buflen, 0);
|
||||
n = send(sd, buf, strlen(buf), 0);
|
||||
|
||||
return n;
|
||||
}
|
||||
@ -369,28 +324,28 @@ int ssl_want_retry(SSL *ssl, int ret, int timeout){
|
||||
i = SSL_get_error(ssl, ret);
|
||||
if(i == SSL_ERROR_NONE)
|
||||
return 1;
|
||||
|
||||
|
||||
tv.tv_sec = timeout/1000;
|
||||
tv.tv_usec = 0;
|
||||
FD_ZERO(&fds);
|
||||
|
||||
|
||||
switch(i){
|
||||
case SSL_ERROR_WANT_READ: // pause until the socket is readable
|
||||
sock = SSL_get_rfd(ssl);
|
||||
FD_SET(sock, &fds);
|
||||
i = select(sock+1, &fds, 0, 0, &tv);
|
||||
break;
|
||||
|
||||
|
||||
case SSL_ERROR_WANT_WRITE: // pause until the socket is writeable
|
||||
sock = SSL_get_wfd(ssl);
|
||||
FD_SET(sock, &fds);
|
||||
i = select(sock+1, 0, &fds, 0, &tv);
|
||||
break;
|
||||
|
||||
|
||||
case SSL_ERROR_ZERO_RETURN: // the sock closed, just return quietly
|
||||
i = 0;
|
||||
break;
|
||||
|
||||
|
||||
default: // ERROR - unexpected error code
|
||||
i = -1;
|
||||
break;
|
||||
@ -414,31 +369,19 @@ int ssl_read_timeout(SSL *ssl, void *buf, int len, int timeout){
|
||||
}
|
||||
|
||||
|
||||
int recvtimeoutssl(struct net *net, char *buf, int len){
|
||||
int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl){
|
||||
|
||||
memset(buf, 0, len);
|
||||
|
||||
if(net->use_ssl == 1){
|
||||
return ssl_read_timeout(net->ssl, buf, len-1, net->timeout);
|
||||
if(use_ssl == 1){
|
||||
return ssl_read_timeout(ssl, buf, len-1, timeout);
|
||||
}
|
||||
else {
|
||||
return recvtimeout(net->socket, buf, len-1, net->timeout);
|
||||
return recvtimeout(s, buf, len-1, timeout);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void close_connection(struct net *net){
|
||||
close(net->socket);
|
||||
|
||||
if(net->use_ssl == 1){
|
||||
SSL_shutdown(net->ssl);
|
||||
SSL_free(net->ssl);
|
||||
SSL_CTX_free(net->ctx);
|
||||
ERR_free_strings();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void write_pid_file(char *pidfile){
|
||||
FILE *f;
|
||||
|
||||
@ -469,7 +412,31 @@ int drop_privileges(struct passwd *pwd){
|
||||
}
|
||||
|
||||
|
||||
void init_session_data(struct session_data *sdata, struct config *cfg){
|
||||
int is_email_address_on_my_domains(char *email, struct __data *data){
|
||||
int rc=0;
|
||||
char *p, *q=NULL;
|
||||
|
||||
if(email == NULL || data->mydomains[0] == '\0') return rc;
|
||||
|
||||
p = strchr(email, '@');
|
||||
|
||||
if(!p) return rc;
|
||||
|
||||
if(strlen(p) < 3) return rc;
|
||||
|
||||
q = strrchr(p+1, ' ');
|
||||
|
||||
if(q) *q = '\0';
|
||||
|
||||
if(strcasestr(data->mydomains, p+1)) rc = 1;
|
||||
|
||||
if(q) *q = ' ';
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
void init_session_data(struct session_data *sdata, struct __config *cfg){
|
||||
int i;
|
||||
|
||||
|
||||
@ -490,8 +457,6 @@ void init_session_data(struct session_data *sdata, struct config *cfg){
|
||||
|
||||
memset(sdata->fromemail, 0, SMALLBUFSIZE);
|
||||
|
||||
sdata->duplicate_id = 0;
|
||||
|
||||
sdata->restored_copy = 0;
|
||||
|
||||
sdata->internal_sender = sdata->internal_recipient = sdata->external_recipient = 0;
|
||||
@ -499,7 +464,6 @@ void init_session_data(struct session_data *sdata, struct config *cfg){
|
||||
|
||||
sdata->hdr_len = 0;
|
||||
sdata->tot_len = 0;
|
||||
sdata->stored_len = 0;
|
||||
sdata->num_of_rcpt_to = 0;
|
||||
|
||||
sdata->ms_journal = 0;
|
||||
@ -512,19 +476,13 @@ void init_session_data(struct session_data *sdata, struct config *cfg){
|
||||
|
||||
sdata->spam_message = 0;
|
||||
|
||||
sdata->customer_id = 0;
|
||||
|
||||
sdata->__acquire = sdata->__parsed = sdata->__av = sdata->__store = sdata->__compress = sdata->__encrypt = 0;
|
||||
|
||||
sdata->import = 0;
|
||||
|
||||
for(i=0; i<MAX_RCPT_TO; i++) memset(sdata->rcptto[i], 0, SMALLBUFSIZE);
|
||||
|
||||
time(&(sdata->now));
|
||||
sdata->delivered = sdata->retained = sdata->now;
|
||||
sdata->sent = 0;
|
||||
|
||||
sdata->sql_errno = 0;
|
||||
sdata->sent = sdata->delivered = sdata->retained = sdata->now;
|
||||
|
||||
#ifdef HAVE_TWEAK_SENT_TIME
|
||||
sdata->sent += cfg->tweak_sent_time_offset;
|
||||
@ -558,180 +516,7 @@ int read_from_stdin(struct session_data *sdata){
|
||||
|
||||
|
||||
void strtolower(char *s){
|
||||
for(; *s; s++){
|
||||
if(*s >= 65 && *s <= 90) *s = tolower(*s);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int make_socket_non_blocking(int fd){
|
||||
int flags, s;
|
||||
|
||||
flags = fcntl(fd, F_GETFL, 0);
|
||||
if(flags == -1){
|
||||
return -1;
|
||||
}
|
||||
|
||||
flags |= O_NONBLOCK;
|
||||
|
||||
s = fcntl(fd, F_SETFL, flags);
|
||||
if(s == -1){
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int create_and_bind(char *listen_addr, int listen_port){
|
||||
struct addrinfo hints;
|
||||
struct addrinfo *result, *rp;
|
||||
char port_string[8];
|
||||
int rc, fd;
|
||||
|
||||
memset(&hints, 0, sizeof (struct addrinfo));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
hints.ai_flags = AI_PASSIVE;
|
||||
|
||||
snprintf(port_string, sizeof(port_string)-1, "%d", listen_port);
|
||||
|
||||
rc = getaddrinfo(listen_addr, port_string, &hints, &result);
|
||||
if(rc != 0){
|
||||
syslog(LOG_PRIORITY, "getaddrinfo for '%s': %s", listen_addr, gai_strerror(rc));
|
||||
return -1;
|
||||
}
|
||||
|
||||
for(rp = result; rp != NULL; rp = rp->ai_next){
|
||||
fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
|
||||
if(fd == -1)
|
||||
continue;
|
||||
|
||||
rc = bind(fd, rp->ai_addr, rp->ai_addrlen);
|
||||
if(rc == 0){
|
||||
break;
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
|
||||
if(rp == NULL){
|
||||
syslog(LOG_PRIORITY, "cannot bind to port: %s:%d", listen_addr, listen_port);
|
||||
return -1;
|
||||
}
|
||||
|
||||
freeaddrinfo(result);
|
||||
|
||||
return fd;
|
||||
}
|
||||
|
||||
|
||||
int can_i_write_directory(char *dir){
|
||||
int fd;
|
||||
char filename[SMALLBUFSIZE];
|
||||
|
||||
if(dir)
|
||||
snprintf(filename, sizeof(filename)-1, "%s/__piler_%d", dir, getpid());
|
||||
else
|
||||
snprintf(filename, sizeof(filename)-1, "__piler_%d", getpid());
|
||||
|
||||
fd = open(filename, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP);
|
||||
if(fd == -1){
|
||||
return 0;
|
||||
}
|
||||
|
||||
close(fd);
|
||||
unlink(filename);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
void move_email(struct smtp_session *session){
|
||||
char buf[SMALLBUFSIZE];
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[0] % session->cfg->number_of_worker_processes, session->ttmpfile);
|
||||
|
||||
if(session->nullbyte){
|
||||
snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, session->ttmpfile);
|
||||
syslog(LOG_PRIORITY, "ERROR: %s contains an invalid NUL-byte, moving it to %s", session->ttmpfile, ERROR_DIR);
|
||||
}
|
||||
|
||||
if(rename(session->ttmpfile, buf)){
|
||||
syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s (reason: %s)", session->ttmpfile, buf, strerror(errno));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte){
|
||||
int i=0;
|
||||
|
||||
*rc = ERR;
|
||||
*nullbyte = 0;
|
||||
|
||||
memset(buf, 0, buflen);
|
||||
|
||||
for(int j=0; j<slen; j++){
|
||||
if(i<buflen-2){
|
||||
if(*(s+j) == 0){
|
||||
*nullbyte = 1;
|
||||
}
|
||||
|
||||
buf[i] = *(s+j);
|
||||
i++;
|
||||
|
||||
if(*(s+j) == c){
|
||||
*rc = OK;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return i;
|
||||
}
|
||||
|
||||
|
||||
int init_ssl_to_server(struct data *data){
|
||||
int n;
|
||||
X509* server_cert;
|
||||
char *str;
|
||||
|
||||
SSL_library_init();
|
||||
SSL_load_error_strings();
|
||||
|
||||
#if OPENSSL_VERSION_NUMBER < 0x10100000L
|
||||
data->net->ctx = SSL_CTX_new(TLSv1_2_client_method());
|
||||
#else
|
||||
data->net->ctx = SSL_CTX_new(TLS_client_method());
|
||||
#endif
|
||||
CHK_NULL(data->net->ctx, "internal SSL error");
|
||||
|
||||
data->net->ssl = SSL_new(data->net->ctx);
|
||||
CHK_NULL(data->net->ssl, "internal ssl error");
|
||||
|
||||
SSL_set_fd(data->net->ssl, data->net->socket);
|
||||
n = SSL_connect(data->net->ssl);
|
||||
CHK_SSL(n, "internal ssl error");
|
||||
|
||||
//printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
|
||||
|
||||
server_cert = SSL_get_peer_certificate(data->net->ssl);
|
||||
CHK_NULL(server_cert, "server cert error");
|
||||
|
||||
str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
|
||||
CHK_NULL(str, "error in server cert");
|
||||
OPENSSL_free(str);
|
||||
|
||||
str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
|
||||
CHK_NULL(str, "error in server cert");
|
||||
OPENSSL_free(str);
|
||||
|
||||
X509_free(server_cert);
|
||||
|
||||
return OK;
|
||||
for(; *s; s++) *s = tolower(*s);
|
||||
}
|
||||
|
||||
|
||||
@ -756,54 +541,3 @@ char *strcasestr(const char *s, const char *find){
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
int append_string_to_buffer(char **buffer, char *str){
|
||||
int arglen;
|
||||
char *s=NULL;
|
||||
|
||||
arglen = strlen(str);
|
||||
|
||||
if(!*buffer){
|
||||
*buffer = malloc(arglen+1);
|
||||
memset(*buffer, 0, arglen+1);
|
||||
memcpy(*buffer, str, arglen);
|
||||
}
|
||||
else {
|
||||
int len = strlen(*buffer);
|
||||
s = realloc(*buffer, len + arglen+1);
|
||||
if(!s) return 1;
|
||||
|
||||
*buffer = s;
|
||||
|
||||
memset(*buffer+len, 0, arglen+1);
|
||||
memcpy(*buffer+len, str, arglen);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int get_size_from_smtp_mail_from(char *s){
|
||||
int size=0;
|
||||
char *p;
|
||||
|
||||
p = strcasestr(s, "SIZE=");
|
||||
if(p){
|
||||
p += strlen("SIZE=");
|
||||
char *q = p;
|
||||
for(; *q; q++){
|
||||
if(isspace(*q)) break;
|
||||
}
|
||||
|
||||
// We extract max. 9 characters, which is just under 1GB
|
||||
// and not overflowing an int variable
|
||||
if(q - p <= 9){
|
||||
char c = *q;
|
||||
*q = '\0';
|
||||
size = atoi(p);
|
||||
*q = c;
|
||||
}
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
32
src/misc.h
32
src/misc.h
@ -5,7 +5,6 @@
|
||||
#ifndef _MISC_H
|
||||
#define _MISC_H
|
||||
|
||||
#include <sys/socket.h>
|
||||
#include <openssl/ssl.h>
|
||||
#include <sys/time.h>
|
||||
#include <pwd.h>
|
||||
@ -16,45 +15,32 @@
|
||||
#define CHK_SSL(err, msg) if ((err)==-1) { printf("ssl error: %s\n", msg); return ERR; }
|
||||
|
||||
int get_build();
|
||||
void get_extractor_list();
|
||||
void __fatal(char *s);
|
||||
long tvdiff(struct timeval a, struct timeval b);
|
||||
char *split(char *str, int ch, char *buf, int buflen, int *result);
|
||||
int searchStringInBuffer(char *s, int len1, char *what, int len2);
|
||||
int countCharacterInBuffer(char *p, char c);
|
||||
void replaceCharacterInBuffer(char *p, char from, char to);
|
||||
char *split(char *row, int ch, char *s, int size);
|
||||
char *split_str(char *row, char *what, char *s, int size);
|
||||
int trimBuffer(char *s);
|
||||
int extractEmail(char *rawmail, char *email);
|
||||
int extract_verp_address(char *email);
|
||||
void make_random_string(unsigned char *buf, int buflen);
|
||||
void create_id(char *id, unsigned char server_id);
|
||||
int get_random_bytes(unsigned char *buf, int len, unsigned char server_id);
|
||||
int readFromEntropyPool(int fd, void *_s, ssize_t n);
|
||||
int readFromEntropyPool(int fd, void *_s, size_t n);
|
||||
int recvtimeout(int s, char *buf, int len, int timeout);
|
||||
int write1(struct net *net, void *buf, int buflen);
|
||||
int recvtimeoutssl(struct net *net, char *buf, int len);
|
||||
void close_connection(struct net *net);
|
||||
int write1(int sd, char *buf, int use_ssl, SSL *ssl);
|
||||
int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl);
|
||||
|
||||
void write_pid_file(char *pidfile);
|
||||
int drop_privileges(struct passwd *pwd);
|
||||
|
||||
void init_session_data(struct session_data *sdata, struct config *cfg);
|
||||
int is_email_address_on_my_domains(char *email, struct __data *data);
|
||||
void init_session_data(struct session_data *sdata, struct __config *cfg);
|
||||
int read_from_stdin(struct session_data *sdata);
|
||||
void strtolower(char *s);
|
||||
|
||||
int make_socket_non_blocking(int fd);
|
||||
int create_and_bind(char *listen_addr, int listen_port);
|
||||
|
||||
int can_i_write_directory(char *dir);
|
||||
|
||||
void move_email(struct smtp_session *session);
|
||||
int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte);
|
||||
|
||||
int init_ssl_to_server(struct data *data);
|
||||
|
||||
#ifndef _GNU_SOURCE
|
||||
char *strcasestr(const char *s, const char *find);
|
||||
#endif
|
||||
|
||||
int append_string_to_buffer(char **buffer, char *str);
|
||||
int get_size_from_smtp_mail_from(char *s);
|
||||
|
||||
#endif /* _MISC_H */
|
||||
|
||||
@ -10,56 +10,35 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg){
|
||||
void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg){
|
||||
int clen=0, len=0, size=sizeof(data->mydomains);
|
||||
char s[SMALLBUFSIZE];
|
||||
struct sql sql;
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
|
||||
memset(s, 0, sizeof(s));
|
||||
memset(data->mydomains, 0, size);
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT `domain` FROM `%s`", SQL_DOMAIN_TABLE);
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_DOMAINS) == ERR) return;
|
||||
if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res != NULL){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
snprintf(s, sizeof(s)-1, "%s,", (char*)row[0]);
|
||||
len = strlen(s);
|
||||
|
||||
if(clen + len + 1 < size){
|
||||
memcpy(data->mydomains+clen, s, len);
|
||||
clen += len;
|
||||
}
|
||||
else break;
|
||||
}
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == OK){
|
||||
|
||||
p_bind_init(&sql);
|
||||
|
||||
sql.sql[sql.pos] = &s[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(s)-2; sql.pos++;
|
||||
|
||||
p_store_results(&sql);
|
||||
|
||||
while(p_fetch_results(&sql) == OK){
|
||||
if(addnode(data->mydomains, s) == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
|
||||
else if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "added mydomain: '%s'", s);
|
||||
|
||||
memset(s, 0, sizeof(s));
|
||||
mysql_free_result(res);
|
||||
}
|
||||
|
||||
p_free_results(&sql);
|
||||
}
|
||||
|
||||
close_prepared_statement(&sql);
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "mydomains: '%s'", data->mydomains);
|
||||
}
|
||||
|
||||
|
||||
int is_email_address_on_my_domains(char *email, struct data *data){
|
||||
int rc=0;
|
||||
char *q, *s;
|
||||
|
||||
if(email == NULL) return rc;
|
||||
|
||||
q = strchr(email, '@');
|
||||
if(!q || strlen(q) < 3) return rc;
|
||||
|
||||
s = strrchr(q+1, ' ');
|
||||
|
||||
if(s) *s = '\0';
|
||||
|
||||
if(findnode(data->mydomains, q+1)) rc = 1;
|
||||
|
||||
if(s) *s = ' ';
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
270
src/mysql.c
270
src/mysql.c
@ -9,253 +9,20 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int open_database(struct session_data *sdata, struct config *cfg){
|
||||
int rc=1;
|
||||
char buf[BUFLEN];
|
||||
int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s){
|
||||
|
||||
mysql_init(&(sdata->mysql));
|
||||
|
||||
mysql_options(&(sdata->mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg->mysql_connect_timeout);
|
||||
mysql_options(&(sdata->mysql), MYSQL_OPT_RECONNECT, (const char*)&rc);
|
||||
|
||||
if(mysql_real_connect(&(sdata->mysql), cfg->mysqlhost, cfg->mysqluser, cfg->mysqlpwd, cfg->mysqldb, cfg->mysqlport, cfg->mysqlsocket, 0) == 0){
|
||||
syslog(LOG_PRIORITY, "cant connect to mysql server");
|
||||
*stmt = mysql_stmt_init(&(sdata->mysql));
|
||||
if(!*stmt){
|
||||
syslog(LOG_PRIORITY, "%s: mysql_stmt_init() error", sdata->ttmpfile);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
|
||||
mysql_real_query(&(sdata->mysql), buf, strlen(buf));
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
|
||||
mysql_real_query(&(sdata->mysql), buf, strlen(buf));
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
int open_sphx(struct session_data *sdata, struct config *cfg){
|
||||
int rc=1;
|
||||
char buf[BUFLEN];
|
||||
|
||||
mysql_init(&(sdata->sphx));
|
||||
|
||||
mysql_options(&(sdata->sphx), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg->mysql_connect_timeout);
|
||||
mysql_options(&(sdata->sphx), MYSQL_OPT_RECONNECT, (const char*)&rc);
|
||||
|
||||
if(mysql_real_connect(&(sdata->sphx), cfg->sphxhost, "", "", cfg->sphxdb, cfg->sphxport, "", 0) == 0){
|
||||
syslog(LOG_PRIORITY, "cant connect to %s:%d", cfg->sphxhost, cfg->sphxport);
|
||||
if(mysql_stmt_prepare(*stmt, s, strlen(s))){
|
||||
syslog(LOG_PRIORITY, "%s: mysql_stmt_prepare() error: %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(*stmt), s);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
|
||||
mysql_real_query(&(sdata->sphx), buf, strlen(buf));
|
||||
|
||||
snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
|
||||
mysql_real_query(&(sdata->sphx), buf, strlen(buf));
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
void close_database(struct session_data *sdata){
|
||||
mysql_close(&(sdata->mysql));
|
||||
}
|
||||
|
||||
|
||||
void close_sphx(struct session_data *sdata){
|
||||
mysql_close(&(sdata->sphx));
|
||||
}
|
||||
|
||||
|
||||
void p_bind_init(struct sql *sql){
|
||||
int i;
|
||||
|
||||
sql->pos = 0;
|
||||
|
||||
for(i=0; i<MAX_SQL_VARS; i++){
|
||||
sql->sql[i] = NULL;
|
||||
sql->type[i] = TYPE_UNDEF;
|
||||
sql->len[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void p_query(struct session_data *sdata, char *s){
|
||||
mysql_real_query(&(sdata->mysql), s, strlen(s));
|
||||
}
|
||||
|
||||
|
||||
int p_exec_stmt(struct session_data *sdata, struct sql *sql){
|
||||
MYSQL_BIND bind[MAX_SQL_VARS];
|
||||
unsigned long length[MAX_SQL_VARS];
|
||||
int i, ret=ERR;
|
||||
|
||||
sdata->sql_errno = 0;
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
for(i=0; i<MAX_SQL_VARS; i++){
|
||||
if(sql->type[i] > TYPE_UNDEF){
|
||||
|
||||
|
||||
switch(sql->type[i]) {
|
||||
case TYPE_SHORT:
|
||||
bind[i].buffer_type = MYSQL_TYPE_SHORT;
|
||||
bind[i].length = 0;
|
||||
break;
|
||||
|
||||
case TYPE_LONG:
|
||||
bind[i].buffer_type = MYSQL_TYPE_LONG;
|
||||
bind[i].length = 0;
|
||||
break;
|
||||
|
||||
|
||||
case TYPE_LONGLONG:
|
||||
bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
bind[i].length = 0;
|
||||
break;
|
||||
|
||||
|
||||
case TYPE_STRING:
|
||||
bind[i].buffer_type = MYSQL_TYPE_STRING;
|
||||
length[i] = strlen(sql->sql[i]);
|
||||
bind[i].length = &length[i];
|
||||
break;
|
||||
|
||||
|
||||
default:
|
||||
bind[i].buffer_type = 0;
|
||||
bind[i].length = 0;
|
||||
break;
|
||||
|
||||
};
|
||||
|
||||
|
||||
bind[i].buffer = sql->sql[i];
|
||||
bind[i].is_null = 0;
|
||||
|
||||
}
|
||||
else { break; }
|
||||
}
|
||||
|
||||
if(mysql_stmt_bind_param(sql->stmt, bind)){
|
||||
sdata->sql_errno = mysql_stmt_errno(sql->stmt);
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: mysql_stmt_bind_param() '%s' (errno: %d)", sdata->ttmpfile, mysql_stmt_error(sql->stmt), sdata->sql_errno);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if(mysql_stmt_execute(sql->stmt)){
|
||||
sdata->sql_errno = mysql_stmt_errno(sql->stmt);
|
||||
syslog(LOG_PRIORITY, "ERROR: %s: mysql_stmt_execute() '%s' (errno: %d)", sdata->ttmpfile, mysql_error(&(sdata->mysql)), sdata->sql_errno);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = OK;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int p_store_results(struct sql *sql){
|
||||
MYSQL_BIND bind[MAX_SQL_VARS];
|
||||
int i, ret=ERR;
|
||||
|
||||
memset(bind, 0, sizeof(bind));
|
||||
|
||||
for(i=0; i<MAX_SQL_VARS; i++){
|
||||
if(sql->type[i] > TYPE_UNDEF){
|
||||
|
||||
switch(sql->type[i]) {
|
||||
case TYPE_SHORT: bind[i].buffer_type = MYSQL_TYPE_SHORT;
|
||||
break;
|
||||
|
||||
|
||||
case TYPE_LONG: bind[i].buffer_type = MYSQL_TYPE_LONG;
|
||||
break;
|
||||
|
||||
|
||||
case TYPE_LONGLONG:
|
||||
bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
|
||||
break;
|
||||
|
||||
|
||||
case TYPE_STRING:
|
||||
bind[i].buffer_type = MYSQL_TYPE_STRING;
|
||||
bind[i].buffer_length = sql->len[i];
|
||||
break;
|
||||
|
||||
default:
|
||||
bind[i].buffer_type = 0;
|
||||
break;
|
||||
|
||||
};
|
||||
|
||||
|
||||
bind[i].buffer = (char *)sql->sql[i];
|
||||
bind[i].is_null = &(sql->is_null[i]);
|
||||
bind[i].length = &(sql->length[i]);
|
||||
bind[i].error = &(sql->error[i]);
|
||||
|
||||
}
|
||||
else { break; }
|
||||
}
|
||||
|
||||
if(mysql_stmt_bind_result(sql->stmt, bind)){
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
if(mysql_stmt_store_result(sql->stmt)){
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = OK;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int p_fetch_results(struct sql *sql){
|
||||
|
||||
if(mysql_stmt_fetch(sql->stmt) == 0) return OK;
|
||||
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
void p_free_results(struct sql *sql){
|
||||
mysql_stmt_free_result(sql->stmt);
|
||||
}
|
||||
|
||||
|
||||
uint64 p_get_insert_id(struct sql *sql){
|
||||
return mysql_stmt_insert_id(sql->stmt);
|
||||
}
|
||||
|
||||
|
||||
int p_get_affected_rows(struct sql *sql){
|
||||
return mysql_stmt_affected_rows(sql->stmt);
|
||||
}
|
||||
|
||||
|
||||
int prepare_sql_statement(struct session_data *sdata, struct sql *sql, char *s){
|
||||
|
||||
sql->stmt = mysql_stmt_init(&(sdata->mysql));
|
||||
if(!(sql->stmt)){
|
||||
syslog(LOG_PRIORITY, "%s: error: mysql_stmt_init()", sdata->ttmpfile);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(mysql_stmt_prepare(sql->stmt, s, strlen(s))){
|
||||
syslog(LOG_PRIORITY, "%s: error: mysql_stmt_prepare() %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(sql->stmt), s);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
void close_prepared_statement(struct sql *sql){
|
||||
if(sql->stmt) mysql_stmt_close(sql->stmt);
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
@ -267,3 +34,26 @@ void insert_offset(struct session_data *sdata, int server_id){
|
||||
|
||||
mysql_real_query(&(sdata->mysql), s, strlen(s));
|
||||
}
|
||||
|
||||
|
||||
int create_prepared_statements(struct session_data *sdata, struct __data *data){
|
||||
int rc = OK;
|
||||
|
||||
data->stmt_get_meta_id_by_message_id = NULL;
|
||||
data->stmt_insert_into_rcpt_table = NULL;
|
||||
data->stmt_update_metadata_reference = NULL;
|
||||
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_get_meta_id_by_message_id), SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID) == ERR) rc = ERR;
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_rcpt_table), SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE) == ERR) rc = ERR;
|
||||
if(prepare_a_mysql_statement(sdata, &(data->stmt_update_metadata_reference), SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE) == ERR) rc = ERR;
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
void close_prepared_statements(struct __data *data){
|
||||
if(data->stmt_get_meta_id_by_message_id) mysql_stmt_close(data->stmt_get_meta_id_by_message_id);
|
||||
if(data->stmt_insert_into_rcpt_table) mysql_stmt_close(data->stmt_insert_into_rcpt_table);
|
||||
if(data->stmt_update_metadata_reference) mysql_stmt_close(data->stmt_update_metadata_reference);
|
||||
}
|
||||
|
||||
|
||||
655
src/parser.c
655
src/parser.c
@ -16,11 +16,12 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg){
|
||||
struct _state parse_message(struct session_data *sdata, int take_into_pieces, struct __data *data, struct __config *cfg){
|
||||
FILE *f;
|
||||
char buf[MAXBUFSIZE];
|
||||
int i, len;
|
||||
char *p, buf[MAXBUFSIZE], puf[SMALLBUFSIZE];
|
||||
char writebuffer[MAXBUFSIZE], abuffer[MAXBUFSIZE];
|
||||
struct parser_state state;
|
||||
struct _state state;
|
||||
|
||||
init_state(&state);
|
||||
|
||||
@ -30,6 +31,45 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
|
||||
return state;
|
||||
}
|
||||
|
||||
|
||||
if(sdata->num_of_rcpt_to > 0){
|
||||
for(i=0; i<sdata->num_of_rcpt_to; i++){
|
||||
extractEmail(sdata->rcptto[i], puf);
|
||||
|
||||
if(strlen(puf) > 5){
|
||||
p = strstr(puf, cfg->hostid);
|
||||
if(p && *(p-1) == '.'){
|
||||
|
||||
*(p-1) = ' ';
|
||||
*p = '\0';
|
||||
|
||||
len = strlen(puf);
|
||||
|
||||
if(does_it_seem_like_an_email_address(puf) == 1){
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: processing rcpt to address: *%s*", sdata->ttmpfile, puf);
|
||||
|
||||
if(state.tolen < MAXBUFSIZE-len-1){
|
||||
if(is_string_on_list(state.rcpt, puf) == 0){
|
||||
append_list(&(state.rcpt), puf);
|
||||
memcpy(&(state.b_to[state.tolen]), puf, len);
|
||||
state.tolen += len;
|
||||
|
||||
if(state.tolen < MAXBUFSIZE-len-1){
|
||||
split_email_address(puf);
|
||||
memcpy(&(state.b_to[state.tolen]), puf, len);
|
||||
state.tolen += len;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if(take_into_pieces == 1){
|
||||
state.mfd = open(sdata->tmpframe, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
|
||||
if(state.mfd == -1){
|
||||
@ -44,70 +84,33 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
|
||||
}
|
||||
|
||||
if(take_into_pieces == 1 && state.writebufpos > 0){
|
||||
if(write(state.mfd, writebuffer, state.writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: %s: write(), %s, %d, %s", sdata->ttmpfile, __func__, __LINE__, __FILE__);
|
||||
memset(writebuffer, 0, sizeof(writebuffer)); //-V597
|
||||
len = write(state.mfd, writebuffer, state.writebufpos);
|
||||
memset(writebuffer, 0, sizeof(writebuffer));
|
||||
state.writebufpos = 0;
|
||||
}
|
||||
|
||||
if(take_into_pieces == 1){
|
||||
close(state.mfd); state.mfd = 0;
|
||||
|
||||
if(state.has_to_dump_whole_body == 1){
|
||||
if(state.abufpos > 0){
|
||||
flush_attachment_buffer(&state, &abuffer[0], sizeof(abuffer));
|
||||
}
|
||||
if(state.fd != -1) close(state.fd);
|
||||
if(state.b64fd != -1) close(state.b64fd);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
|
||||
if(data->import && data->import->extra_recipient){
|
||||
char tmpbuf[SMALLBUFSIZE];
|
||||
snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s", data->import->extra_recipient);
|
||||
add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
|
||||
}
|
||||
|
||||
// If both Sender: and From: headers exist, and they are different, then append
|
||||
// the From: address to recipients list to give him access to this email as well
|
||||
|
||||
if(state.b_sender_domain[0]){
|
||||
char frombuf[SMALLBUFSIZE];
|
||||
char senderbuf[SMALLBUFSIZE];
|
||||
get_first_email_address_from_string(state.b_from, frombuf, sizeof(frombuf));
|
||||
get_first_email_address_from_string(state.b_sender, senderbuf, sizeof(senderbuf));
|
||||
if(strcmp(frombuf, senderbuf)){
|
||||
frombuf[strlen(frombuf)] = ' ';
|
||||
add_recipient(frombuf, strlen(frombuf), sdata, &state, data, cfg);
|
||||
}
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
|
||||
void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){
|
||||
int i;
|
||||
void post_parse(struct session_data *sdata, struct _state *state, struct __config *cfg){
|
||||
int i, len, rec=0;
|
||||
char *p;
|
||||
|
||||
clearhash(state->boundaries);
|
||||
clearhash(state->rcpt);
|
||||
clearhash(state->rcpt_domain);
|
||||
clearhash(state->journal_recipient);
|
||||
free_list(state->boundaries);
|
||||
free_list(state->rcpt);
|
||||
free_list(state->rcpt_domain);
|
||||
free_list(state->journal_recipient);
|
||||
|
||||
// Fix From: and Sender: lines if they are too long
|
||||
if(strlen(state->b_from) > 255) state->b_from[255] = '\0';
|
||||
if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0';
|
||||
|
||||
if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0';
|
||||
if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0';
|
||||
|
||||
// Truncate the message_id if it's >255 characters
|
||||
if(strlen(state->message_id) > 255) state->message_id[255] = '\0';
|
||||
|
||||
fixupEncodedHeaderLine(state->b_subject, MAXBUFSIZE);
|
||||
trimBuffer(state->b_subject);
|
||||
fixupEncodedHeaderLine(state->b_subject);
|
||||
|
||||
|
||||
if(sdata->internal_sender == 0) sdata->direction = DIRECTION_INCOMING;
|
||||
else {
|
||||
@ -116,50 +119,20 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
|
||||
if(sdata->internal_recipient == 1 && sdata->external_recipient == 1) sdata->direction = DIRECTION_INTERNAL_AND_OUTGOING;
|
||||
}
|
||||
|
||||
char *q = strrchr(state->receivedbuf, ';');
|
||||
// I've seen some odd 1st Received: lines where the date
|
||||
// wasn't the last item after a semicolon, eg.
|
||||
//
|
||||
// Received: from some.server.local [172.16.102.18]
|
||||
// by some.server.com with POP3 (Message Archiver)
|
||||
// for <nobody@127.0.0.1> (single-drop); Fri, 22 Nov 2019 11:24:49 -0500 (EST)
|
||||
// smtp.auth=3Daaaaaaaa@yourdomain.com|Received:=20by=20yourdomain
|
||||
// .com=20with=20ESMTPA=20id=20md50124862414.msg=3B=20Fri,=2022=20
|
||||
// ...
|
||||
//
|
||||
// Such line messes with the parse_date_header() function
|
||||
// so the workaround is to discard such Received: line
|
||||
if(q && strlen(q+1) < 45){
|
||||
time_t received_timestamp = parse_date_header(q+1);
|
||||
if(received_timestamp > 10000000){
|
||||
// If the calculated date based on Date: header line differs more than 1 week
|
||||
// then we'll override it with the data parsed from the first Received: line
|
||||
if(labs(received_timestamp - sdata->sent) > 604800) sdata->sent = received_timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
for(i=1; i<=state->n_attachments; i++){
|
||||
char puf[SMALLBUFSIZE];
|
||||
snprintf(puf, sizeof(puf)-1, "%s ", state->attachments[i].filename);
|
||||
|
||||
unsigned int len = strlen(puf);
|
||||
if(state->bodylen < BIGBUFSIZE-len-1){
|
||||
memcpy(&(state->b_body[state->bodylen]), puf, len);
|
||||
state->bodylen += len;
|
||||
}
|
||||
|
||||
digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0]));
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s, dumped: %d", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest, state->attachments[i].dumped);
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest);
|
||||
|
||||
char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
|
||||
p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
|
||||
len = strlen(p);
|
||||
if(strlen(sdata->attachments) < SMALLBUFSIZE-len-1 && !strstr(sdata->attachments, p)) memcpy(&(sdata->attachments[strlen(sdata->attachments)]), p, len);
|
||||
|
||||
if(state->attachments[i].dumped == 1){
|
||||
int rec = 0;
|
||||
if(cfg->extract_attachments == 1 && state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec, cfg);
|
||||
|
||||
rec = 0;
|
||||
if(state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec);
|
||||
|
||||
unlink(state->attachments[i].aname);
|
||||
}
|
||||
|
||||
@ -172,14 +145,10 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
|
||||
else snprintf(state->message_id, SMALLBUFSIZE-1, "null");
|
||||
}
|
||||
|
||||
|
||||
digest_string("sha256", state->message_id, &(state->message_id_hash[0]));
|
||||
|
||||
if(sdata->sent == 0) sdata->sent = sdata->now;
|
||||
}
|
||||
|
||||
|
||||
void storno_attachment(struct parser_state *state){
|
||||
void storno_attachment(struct _state *state){
|
||||
state->has_to_dump = 0;
|
||||
|
||||
if(state->n_attachments <= 0) return;
|
||||
@ -190,7 +159,7 @@ void storno_attachment(struct parser_state *state){
|
||||
memset(state->attachments[state->n_attachments].type, 0, TINYBUFSIZE);
|
||||
memset(state->attachments[state->n_attachments].shorttype, 0, TINYBUFSIZE);
|
||||
memset(state->attachments[state->n_attachments].aname, 0, TINYBUFSIZE);
|
||||
memset(state->attachments[state->n_attachments].filename, 0, SMALLBUFSIZE);
|
||||
memset(state->attachments[state->n_attachments].filename, 0, TINYBUFSIZE);
|
||||
memset(state->attachments[state->n_attachments].internalname, 0, TINYBUFSIZE);
|
||||
memset(state->attachments[state->n_attachments].digest, 0, 2*DIGEST_LENGTH+1);
|
||||
|
||||
@ -199,70 +168,30 @@ void storno_attachment(struct parser_state *state){
|
||||
}
|
||||
|
||||
|
||||
void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize){
|
||||
if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
|
||||
|
||||
if(state->b64fd != -1){
|
||||
abuffer[state->abufpos] = '\0';
|
||||
if(state->base64 == 1){
|
||||
unsigned char b64buffer[MAXBUFSIZE];
|
||||
int n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
|
||||
if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
|
||||
}
|
||||
else if(write(state->b64fd, abuffer, state->abufpos) == -1){
|
||||
syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
|
||||
}
|
||||
}
|
||||
|
||||
state->abufpos = 0;
|
||||
memset(abuffer, 0, abuffersize);
|
||||
}
|
||||
|
||||
|
||||
int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg){
|
||||
char *p;
|
||||
int boundary_line=0;
|
||||
unsigned int len;
|
||||
int parse_line(char *buf, struct _state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct __data *data, struct __config *cfg){
|
||||
char *p, *q, puf[SMALLBUFSIZE];
|
||||
unsigned char b64buffer[MAXBUFSIZE];
|
||||
int n64, len, writelen, boundary_line=0;
|
||||
|
||||
if(cfg->debug == 1) printf("line: %s", buf);
|
||||
|
||||
state->line_num++;
|
||||
len = strlen(buf);
|
||||
|
||||
/*
|
||||
* check a few things in the 1st header
|
||||
*/
|
||||
if(state->is_1st_header == 1 && (strncmp(buf, "Received: by piler", strlen("Received: by piler")) == 0 || strncmp(buf, "X-piler-id: ", strlen("X-piler-id: ")) == 0) ){
|
||||
sdata->restored_copy = 1;
|
||||
}
|
||||
|
||||
if(state->is_1st_header == 1){
|
||||
if(state->is_1st_header == 1 && *(cfg->spam_header_line) != '\0' && strncmp(buf, cfg->spam_header_line, strlen(cfg->spam_header_line)) == 0){
|
||||
sdata->spam_message = 1;
|
||||
}
|
||||
|
||||
if(strncmp(buf, "Received: by piler", strlen("Received: by piler")) == 0){
|
||||
sdata->restored_copy = 1;
|
||||
}
|
||||
|
||||
if(cfg->security_header[0] && state->found_security_header == 0 && strstr(buf, cfg->security_header)){
|
||||
state->found_security_header = 1;
|
||||
}
|
||||
|
||||
if(*(cfg->piler_header_field) != 0 && strncmp(buf, cfg->piler_header_field, strlen(cfg->piler_header_field)) == 0){
|
||||
sdata->restored_copy = 1;
|
||||
}
|
||||
|
||||
if(sdata->ms_journal == 0 && ( strncmp(buf, "X-MS-Journal-Report:", strlen("X-MS-Journal-Report:")) == 0 || strncmp(buf, "X-WM-Journal-Report: journal", strlen("X-WM-Journal-Report: journal")) == 0) ){
|
||||
|
||||
memset(state->b_to, 0, MAXBUFSIZE);
|
||||
state->tolen = 0;
|
||||
memset(state->b_to_domain, 0, SMALLBUFSIZE);
|
||||
state->todomainlen = 0;
|
||||
|
||||
clearhash(state->rcpt);
|
||||
clearhash(state->rcpt_domain);
|
||||
|
||||
//if(sdata->import == 0){
|
||||
sdata->ms_journal = 1;
|
||||
memset(state->message_id, 0, SMALLBUFSIZE);
|
||||
//}
|
||||
}
|
||||
if(state->is_1st_header == 1 && sdata->ms_journal == 0 && strncmp(buf, "X-MS-Journal-Report:", strlen("X-MS-Journal-Report:")) == 0){
|
||||
sdata->ms_journal = 1;
|
||||
memset(state->message_id, 0, SMALLBUFSIZE);
|
||||
|
||||
memset(state->b_from, 0, SMALLBUFSIZE);
|
||||
memset(state->b_from_domain, 0, SMALLBUFSIZE);
|
||||
}
|
||||
|
||||
|
||||
@ -273,55 +202,76 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
state->is_1st_header = 0;
|
||||
}
|
||||
|
||||
if(sdata->ms_journal == 1 && strncasecmp(buf, "Received:", strlen("Received:")) == 0){
|
||||
state->is_1st_header = 1;
|
||||
state->is_header = 1;
|
||||
memset(state->b_body, 0, BIGBUFSIZE);
|
||||
state->bodylen = 0;
|
||||
memset(state->b_subject, 0, MAXBUFSIZE);
|
||||
}
|
||||
|
||||
if(take_into_pieces == 1){
|
||||
if(state->message_state == MSG_BODY && state->fd != -1 && (state->has_to_dump_whole_body == 1 || is_substr_in_hash(state->boundaries, buf) == 0) ){
|
||||
if(state->message_state == MSG_BODY && state->fd != -1 && is_item_on_string(state->boundaries, buf) == 0){
|
||||
//n = write(state->fd, buf, len); // WRITE
|
||||
if(len + state->abufpos > abuffersize-1){
|
||||
flush_attachment_buffer(state, abuffer, abuffersize);
|
||||
write(state->fd, abuffer, state->abufpos);
|
||||
|
||||
if(state->b64fd != -1){
|
||||
abuffer[state->abufpos] = '\0';
|
||||
if(state->base64 == 1){
|
||||
n64 = base64_decode_attachment_buffer(abuffer, state->abufpos, &b64buffer[0], sizeof(b64buffer));
|
||||
n64 = write(state->b64fd, b64buffer, n64);
|
||||
}
|
||||
else {
|
||||
n64 = write(state->b64fd, abuffer, state->abufpos);
|
||||
}
|
||||
}
|
||||
|
||||
state->abufpos = 0; memset(abuffer, 0, abuffersize);
|
||||
}
|
||||
memcpy(abuffer+state->abufpos, buf, len); state->abufpos += len;
|
||||
|
||||
state->attachments[state->n_attachments].size += len;
|
||||
|
||||
// When processing the body and writing to an attachment file, then we finish here
|
||||
return 0;
|
||||
}
|
||||
else {
|
||||
state->saved_size += len;
|
||||
//n = write(state->mfd, buf, len); // WRITE
|
||||
if(len + state->writebufpos > writebuffersize-1){
|
||||
if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
|
||||
state->writebufpos = 0;
|
||||
memset(writebuffer, 0, writebuffersize);
|
||||
write(state->mfd, writebuffer, state->writebufpos); state->writebufpos = 0; memset(writebuffer, 0, writebuffersize);
|
||||
}
|
||||
memcpy(writebuffer+state->writebufpos, buf, len); state->writebufpos += len;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(state->message_state == MSG_BODY && state->has_to_dump == 1 && state->pushed_pointer == 0){
|
||||
if(state->message_state == MSG_BODY && state->has_to_dump == 1 && state->pushed_pointer == 0){
|
||||
//printf("####name: %s, type: %s, base64: %d\n", state->filename, state->type, state->base64);
|
||||
|
||||
state->pushed_pointer = 1;
|
||||
|
||||
|
||||
// this is a real attachment to dump, it doesn't have to be base64 encoded!
|
||||
if(state->attachment_name_buf[0] != 0 && strcasestr(state->attachment_name_buf, "name") && strlen(state->type) > 3 && state->n_attachments < MAX_ATTACHMENTS-1){
|
||||
if(strlen(state->filename) > 4 && strlen(state->type) > 3 && state->n_attachments < MAX_ATTACHMENTS-1){
|
||||
state->n_attachments++;
|
||||
|
||||
extractNameFromHeaderLine(state->attachment_name_buf, "name", state->attachments[state->n_attachments].filename, SMALLBUFSIZE);
|
||||
snprintf(state->attachments[state->n_attachments].filename, TINYBUFSIZE-1, "%s", state->filename);
|
||||
snprintf(state->attachments[state->n_attachments].type, TINYBUFSIZE-1, "%s", state->type);
|
||||
snprintf(state->attachments[state->n_attachments].internalname, TINYBUFSIZE-1, "%s.a%d", sdata->ttmpfile, state->n_attachments);
|
||||
snprintf(state->attachments[state->n_attachments].aname, TINYBUFSIZE-1, "%s.a%d.bin", sdata->ttmpfile, state->n_attachments);
|
||||
|
||||
//printf("DUMP FILE: %s\n", state->attachments[state->n_attachments].internalname);
|
||||
|
||||
if(take_into_pieces == 1){
|
||||
state->fd = open(state->attachments[state->n_attachments].internalname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
|
||||
|
||||
fixupEncodedHeaderLine(state->attachments[state->n_attachments].filename, SMALLBUFSIZE);
|
||||
fixupEncodedHeaderLine(state->attachments[state->n_attachments].filename);
|
||||
|
||||
p = get_attachment_extractor_by_filename(state->attachments[state->n_attachments].filename);
|
||||
|
||||
snprintf(state->attachments[state->n_attachments].shorttype, TINYBUFSIZE-1, "%s", p);
|
||||
|
||||
|
||||
if(strcmp("other", p)){
|
||||
state->b64fd = open(state->attachments[state->n_attachments].aname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
|
||||
state->b64fd = open(state->attachments[state->n_attachments].aname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
|
||||
state->attachments[state->n_attachments].dumped = 1;
|
||||
}
|
||||
|
||||
@ -332,16 +282,13 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
syslog(LOG_PRIORITY, "%s: error opening %s", sdata->ttmpfile, state->attachments[state->n_attachments].internalname);
|
||||
}
|
||||
else {
|
||||
char puf[SMALLBUFSIZE];
|
||||
snprintf(puf, sizeof(puf)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, state->n_attachments);
|
||||
int writelen = strlen(puf);
|
||||
//n = write(state->mfd, puf, strlen(puf)); // WRITE
|
||||
writelen = strlen(puf);
|
||||
if(writelen + state->writebufpos > writebuffersize-1){
|
||||
if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
|
||||
state->writebufpos = 0;
|
||||
memset(writebuffer, 0, writebuffersize);
|
||||
write(state->mfd, writebuffer, state->writebufpos); state->writebufpos = 0; memset(writebuffer, 0, writebuffersize);
|
||||
}
|
||||
memcpy(writebuffer+state->writebufpos, puf, writelen);
|
||||
state->writebufpos += writelen;
|
||||
memcpy(writebuffer+state->writebufpos, puf, writelen); state->writebufpos += writelen;
|
||||
}
|
||||
}
|
||||
|
||||
@ -374,123 +321,34 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
if(state->is_header == 0 && buf[0] != ' ' && buf[0] != '\t') state->message_state = MSG_BODY;
|
||||
|
||||
|
||||
// journal fix
|
||||
|
||||
if(state->message_state == MSG_BODY && sdata->ms_journal == 1){
|
||||
state->is_header = 1;
|
||||
state->is_1st_header = 1;
|
||||
}
|
||||
|
||||
|
||||
/* header checks */
|
||||
|
||||
if(state->is_header == 1){
|
||||
|
||||
if(*(cfg->spam_header_line) != '\0' && strncmp(buf, cfg->spam_header_line, strlen(cfg->spam_header_line)) == 0){
|
||||
sdata->spam_message = 1;
|
||||
}
|
||||
if(strncasecmp(buf, "From:", strlen("From:")) == 0) state->message_state = MSG_FROM;
|
||||
|
||||
if(strncasecmp(buf, "X-Piler-Envelope-To:", strlen("X-Piler-Envelope-To:")) == 0){
|
||||
state->message_state = MSG_ENVELOPE_TO;
|
||||
buf += strlen("X-Piler-Envelope-To:");
|
||||
}
|
||||
else if(strncasecmp(buf, "From:", strlen("From:")) == 0){
|
||||
state->message_state = MSG_FROM;
|
||||
buf += strlen("From:");
|
||||
}
|
||||
else if(strncasecmp(buf, "Sender:", strlen("Sender:")) == 0){
|
||||
state->message_state = MSG_SENDER;
|
||||
buf += strlen("Sender:");
|
||||
}
|
||||
else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){
|
||||
state->message_state = MSG_CONTENT_TYPE;
|
||||
}
|
||||
else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0){
|
||||
state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
|
||||
if(state->is_1st_header == 1 && strcasestr(buf, "base64")){
|
||||
state->has_to_dump = 1;
|
||||
state->has_to_dump_whole_body = 1;
|
||||
}
|
||||
}
|
||||
else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0) state->message_state = MSG_CONTENT_TYPE;
|
||||
else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0) state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
|
||||
else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0) state->message_state = MSG_CONTENT_DISPOSITION;
|
||||
|
||||
/*
|
||||
* We only enter MSG_CONTENT_DISPOSITION state if we couldn't find
|
||||
* the filename in MSG_CONTENT_TYPE state. We also assume that
|
||||
* Content-Type: comes first, then Content-Disposition:
|
||||
*/
|
||||
else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0 && strcasestr(state->attachment_name_buf, "name") == NULL){
|
||||
state->message_state = MSG_CONTENT_DISPOSITION;
|
||||
}
|
||||
else if(strncasecmp(buf, "To:", 3) == 0){
|
||||
state->message_state = MSG_TO;
|
||||
buf += strlen("To:");
|
||||
}
|
||||
else if(strncasecmp(buf, "Cc:", 3) == 0){
|
||||
state->message_state = MSG_CC;
|
||||
buf += strlen("Cc:");
|
||||
}
|
||||
else if(strncasecmp(buf, "Bcc:", 4) == 0){
|
||||
state->message_state = MSG_CC;
|
||||
buf += strlen("Bcc:");
|
||||
}
|
||||
else if(strncasecmp(buf, "Message-Id:", 11) == 0){
|
||||
state->message_state = MSG_MESSAGE_ID;
|
||||
buf += strlen("Message-Id:");
|
||||
}
|
||||
else if(strncasecmp(buf, "To:", 3) == 0) state->message_state = MSG_TO;
|
||||
else if(strncasecmp(buf, "Cc:", 3) == 0) state->message_state = MSG_CC;
|
||||
else if(strncasecmp(buf, "Bcc:", 4) == 0) state->message_state = MSG_CC;
|
||||
else if(strncasecmp(buf, "Message-Id:", 11) == 0) state->message_state = MSG_MESSAGE_ID;
|
||||
else if(strncasecmp(buf, "References:", 11) == 0) state->message_state = MSG_REFERENCES;
|
||||
else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0){
|
||||
state->message_state = MSG_SUBJECT;
|
||||
buf += strlen("Subject:");
|
||||
}
|
||||
else if(strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0){
|
||||
state->message_state = MSG_RECIPIENT;
|
||||
buf += strlen("Recipient:");
|
||||
}
|
||||
if(sdata->ms_journal == 1 && (state->message_state == MSG_TO || state->message_state == MSG_RECIPIENT) ){
|
||||
p = strstr(buf, "Expanded:");
|
||||
if(p) *p = '\0';
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* by default sdata->sent = 0, and let the parser extract value from the Date: header
|
||||
*/
|
||||
|
||||
else if(strncasecmp(buf, "Date:", strlen("Date:")) == 0 && state->is_1st_header == 1 && sdata->sent == 0){
|
||||
|
||||
if(strstr(buf, "=?") && strstr(buf, "?=")) fixupEncodedHeaderLine(buf, MAXBUFSIZE);
|
||||
|
||||
sdata->sent = parse_date_header(buf+5);
|
||||
|
||||
/* allow +2 days drift in the parsed Date: value */
|
||||
|
||||
if(sdata->sent - sdata->now > 2*86400) sdata->sent = sdata->now;
|
||||
}
|
||||
|
||||
else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf+14);
|
||||
else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0){
|
||||
state->message_state = MSG_RECEIVED;
|
||||
state->received_header++;
|
||||
}
|
||||
else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0){
|
||||
state->message_state = MSG_RECIPIENT;
|
||||
buf += strlen(cfg->extra_to_field);
|
||||
}
|
||||
else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0) state->message_state = MSG_SUBJECT;
|
||||
else if(strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0) state->message_state = MSG_RECIPIENT;
|
||||
else if(strncasecmp(buf, "Date:", strlen("Date:")) == 0 && sdata->sent == 0) sdata->sent = parse_date_header(buf, cfg);
|
||||
else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf, cfg);
|
||||
else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0) state->message_state = MSG_RECEIVED;
|
||||
else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0) state->message_state = MSG_TO;
|
||||
|
||||
if(state->message_state == MSG_MESSAGE_ID && state->message_id[0] == 0){
|
||||
while(isspace(*buf)){
|
||||
buf++;
|
||||
}
|
||||
p = strchr(buf+11, ' ');
|
||||
if(p) p = buf + 12;
|
||||
else p = buf + 11;
|
||||
|
||||
snprintf(state->message_id, SMALLBUFSIZE-1, "%s", buf);
|
||||
}
|
||||
|
||||
if(state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION){
|
||||
fill_attachment_name_buf(state, buf);
|
||||
}
|
||||
|
||||
if(state->received_header == 1 && state->message_state == MSG_RECEIVED && strlen(state->receivedbuf) + len < sizeof(state->receivedbuf)){
|
||||
memcpy(&(state->receivedbuf[strlen(state->receivedbuf)]), buf, len);
|
||||
snprintf(state->message_id, SMALLBUFSIZE-1, "%s", p);
|
||||
}
|
||||
|
||||
/* we are interested in only From:, To:, Subject:, Received:, Content-*: header lines */
|
||||
@ -505,35 +363,23 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* A normal journal looks like this:
|
||||
*
|
||||
* Sender: sender@domain
|
||||
* Subject: Test normal
|
||||
* Message-Id: ...
|
||||
* Recipient: user1@domain
|
||||
* Recipient: user2@domain, Forwarded: user1@domain
|
||||
*
|
||||
* However if outlook forwards an email, then the journal is somewhat changed:
|
||||
*
|
||||
* Sender: sender@domain
|
||||
* Subject: Test through outlook
|
||||
* Message-Id: ...
|
||||
* To: user1@domain
|
||||
* To: user2@domain, Forwarded: user1@domain
|
||||
*
|
||||
*
|
||||
* Outlook.com has the following scheme, when expanded from a distribution list:
|
||||
*
|
||||
* Sender: sender@domain
|
||||
* Subject: Test Email
|
||||
* Message-Id: ...
|
||||
* To: user1@domain, Expanded: listaddress@domain
|
||||
* To: user2@domain, Expanded: listaddress@domain
|
||||
*
|
||||
*/
|
||||
if(state->message_state == MSG_BODY && sdata->ms_journal == 1 && strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0){
|
||||
state->is_header = 1;
|
||||
state->is_1st_header = 1;
|
||||
state->message_state = MSG_RECIPIENT;
|
||||
}
|
||||
|
||||
if(state->message_state == MSG_BODY && sdata->ms_journal == 1 && strncasecmp(buf, "Bcc:", 4) == 0){
|
||||
state->is_header = 1;
|
||||
state->is_1st_header = 1;
|
||||
state->message_state = MSG_CC;
|
||||
}
|
||||
|
||||
|
||||
if(state->message_state == MSG_RECIPIENT){
|
||||
p = strstr(buf, "Expanded:");
|
||||
if(p) *p = '\0';
|
||||
}
|
||||
|
||||
|
||||
if(state->is_1st_header == 1 && state->message_state == MSG_REFERENCES){
|
||||
@ -542,13 +388,29 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
}
|
||||
|
||||
|
||||
if(state->is_1st_header == 1){
|
||||
if(state->is_1st_header == 1 && state->message_state == MSG_SUBJECT && strlen(state->b_subject) + strlen(buf) < MAXBUFSIZE-1){
|
||||
|
||||
if(state->message_state == MSG_SUBJECT && strlen(state->b_subject) + strlen(buf) < MAXBUFSIZE-1){
|
||||
// buffer the subject lines, and decode it later
|
||||
strncat(state->b_subject, buf, MAXBUFSIZE-strlen(state->b_subject)-1);
|
||||
if(state->b_subject[0] == '\0'){
|
||||
p = &buf[0];
|
||||
if(strncmp(buf, "Subject:", strlen("Subject:")) == 0) p += strlen("Subject:");
|
||||
if(*p == ' ') p++;
|
||||
|
||||
strncat(state->b_subject, p, MAXBUFSIZE-1);
|
||||
}
|
||||
else { fixupEncodedHeaderLine(buf, MAXBUFSIZE); }
|
||||
else {
|
||||
|
||||
p = strrchr(state->b_subject, ' ');
|
||||
if(p && ( strcasestr(p+1, "?Q?") || strcasestr(p+1, "?B?") ) ){
|
||||
strncat(state->b_subject, buf+1, MAXBUFSIZE-1);
|
||||
}
|
||||
else strncat(state->b_subject, buf, MAXBUFSIZE-1);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(state->is_1st_header == 1){
|
||||
fixupEncodedHeaderLine(buf);
|
||||
}
|
||||
|
||||
|
||||
@ -576,7 +438,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
strcasestr(buf, "multipart/report") ||
|
||||
strcasestr(buf, "message/delivery-status") ||
|
||||
strcasestr(buf, "text/rfc822-headers") ||
|
||||
strcasestr(buf, "message/rfc822")
|
||||
strcasestr(buf, "message/rfc822") ||
|
||||
strcasestr(buf, "application/ms-tnef")
|
||||
){
|
||||
state->textplain = 1;
|
||||
}
|
||||
@ -589,29 +452,15 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
if(strcasestr(buf, "message/rfc822")){
|
||||
state->message_rfc822 = 1;
|
||||
state->is_header = 1;
|
||||
|
||||
state->has_to_dump = 0;
|
||||
|
||||
if(sdata->ms_journal == 1){
|
||||
state->is_1st_header = 1;
|
||||
|
||||
// reset all headers, except To:
|
||||
|
||||
memset(state->b_subject, 0, MAXBUFSIZE);
|
||||
memset(state->b_body, 0, BIGBUFSIZE);
|
||||
memset(state->b_from, 0, SMALLBUFSIZE);
|
||||
memset(state->b_from_domain, 0, SMALLBUFSIZE);
|
||||
memset(state->b_sender, 0, SMALLBUFSIZE);
|
||||
memset(state->b_sender_domain, 0, SMALLBUFSIZE);
|
||||
memset(state->message_id, 0, SMALLBUFSIZE);
|
||||
|
||||
sdata->ms_journal = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(strcasestr(buf, "charset")) extractNameFromHeaderLine(buf, "charset", state->charset, TINYBUFSIZE);
|
||||
if(strcasestr(state->charset, "UTF-8")) state->utf8 = 1;
|
||||
if(strcasestr(buf, "charset") && strcasestr(buf, "UTF-8")) state->utf8 = 1;
|
||||
}
|
||||
|
||||
|
||||
if((state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION) && strlen(state->filename) < 5){
|
||||
extractNameFromHeaderLine(buf, "name", state->filename);
|
||||
}
|
||||
|
||||
|
||||
@ -624,8 +473,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
|
||||
/* boundary check, and reset variables */
|
||||
|
||||
boundary_line = is_substr_in_hash(state->boundaries, buf);
|
||||
|
||||
boundary_line = is_item_on_string(state->boundaries, buf);
|
||||
|
||||
if(!strstr(buf, "boundary=") && !strstr(buf, "boundary =") && boundary_line == 1){
|
||||
state->is_header = 1;
|
||||
@ -635,7 +483,20 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
if(state->has_to_dump == 1){
|
||||
if(take_into_pieces == 1 && state->fd != -1){
|
||||
if(state->abufpos > 0){
|
||||
flush_attachment_buffer(state, abuffer, abuffersize);
|
||||
write(state->fd, abuffer, state->abufpos);
|
||||
|
||||
if(state->b64fd != -1){
|
||||
abuffer[state->abufpos] = '\0';
|
||||
if(state->base64 == 1){
|
||||
n64 = base64_decode_attachment_buffer(abuffer, state->abufpos, &b64buffer[0], sizeof(b64buffer));
|
||||
n64 = write(state->b64fd, b64buffer, n64);
|
||||
}
|
||||
else {
|
||||
n64 = write(state->b64fd, abuffer, state->abufpos);
|
||||
}
|
||||
}
|
||||
|
||||
state->abufpos = 0; memset(abuffer, 0, abuffersize);
|
||||
}
|
||||
close(state->fd);
|
||||
close(state->b64fd);
|
||||
@ -656,15 +517,12 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
|
||||
state->pushed_pointer = 0;
|
||||
|
||||
memset(state->filename, 0, TINYBUFSIZE);
|
||||
memset(state->type, 0, TINYBUFSIZE);
|
||||
memset(state->charset, 0, TINYBUFSIZE);
|
||||
|
||||
memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
|
||||
state->anamepos = 0;
|
||||
|
||||
state->message_state = MSG_UNDEF;
|
||||
|
||||
return 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
if(boundary_line == 1){ return 0; }
|
||||
@ -674,7 +532,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
|
||||
|
||||
/* skip irrelevant headers */
|
||||
if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_SENDER && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0;
|
||||
if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT) return 0;
|
||||
|
||||
|
||||
/* don't process body if it's not a text or html part */
|
||||
@ -687,6 +545,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
}
|
||||
|
||||
|
||||
/* remove all HTML tags */
|
||||
if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
|
||||
|
||||
if(state->message_state == MSG_BODY && state->qp == 1){
|
||||
fixupSoftBreakInQuotedPritableLine(buf, state); // 2011.12.07
|
||||
@ -696,31 +556,108 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
|
||||
/* I believe that we can live without this function call */
|
||||
//decodeURL(buf);
|
||||
|
||||
/* remove all HTML tags */
|
||||
if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
|
||||
|
||||
|
||||
if(state->texthtml == 1){
|
||||
size_t buflen = strlen(buf);
|
||||
decodeHTML(buf, state->utf8);
|
||||
/* decodeHTML converted some entities to iso-8859-1 */
|
||||
if(state->utf8 != 1 && strlen(buf) != buflen){
|
||||
/* no charset or us-ascii: switch to iso-8859-1 */
|
||||
if (state->charset[0] == 0 || strcasecmp(state->charset, "us-ascii") == 0){
|
||||
syslog(LOG_PRIORITY, "%s: assuming iso-8859-1 encoding for HTML (was '%s')", sdata->ttmpfile, state->charset);
|
||||
snprintf(state->charset, TINYBUFSIZE-1, "ISO8859-1");
|
||||
}
|
||||
}
|
||||
}
|
||||
if(state->texthtml == 1) decodeHTML(buf);
|
||||
|
||||
/* encode the body if it's not utf-8 encoded */
|
||||
if(state->message_state == MSG_BODY && state->utf8 != 1){
|
||||
char tmpbuf[MAXBUFSIZE];
|
||||
int result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
|
||||
if(result == OK) snprintf(buf, MAXBUFSIZE-1, "%s", tmpbuf);
|
||||
}
|
||||
if(state->message_state == MSG_BODY && state->utf8 != 1) utf8_encode((unsigned char*)buf);
|
||||
|
||||
tokenize(buf, state, sdata, data, cfg);
|
||||
|
||||
translateLine((unsigned char*)buf, state);
|
||||
|
||||
reassembleToken(buf);
|
||||
|
||||
|
||||
if(state->is_header == 1) p = strchr(buf, ' ');
|
||||
else p = buf;
|
||||
|
||||
//printf("a: %d/%d/%d/%d %s\n", state->is_1st_header, state->is_header, state->message_rfc822, state->message_state, buf);
|
||||
|
||||
do {
|
||||
memset(puf, 0, sizeof(puf));
|
||||
p = split(p, ' ', puf, sizeof(puf)-1);
|
||||
|
||||
if(puf[0] == '\0') continue;
|
||||
|
||||
degenerateToken((unsigned char*)puf);
|
||||
|
||||
if(puf[0] == '\0') continue;
|
||||
|
||||
strncat(puf, " ", sizeof(puf)-1);
|
||||
|
||||
if(strncasecmp(puf, "http://", 7) == 0 || strncasecmp(puf, "https://", 8) == 0) fixURL(puf);
|
||||
|
||||
if(state->is_header == 0 && strncmp(puf, "__URL__", 7) && (puf[0] == ' ' || strlen(puf) > MAX_WORD_LEN || isHexNumber(puf)) ) continue;
|
||||
|
||||
|
||||
len = strlen(puf);
|
||||
|
||||
|
||||
if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
|
||||
memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
|
||||
|
||||
if(does_it_seem_like_an_email_address(puf) == 1 && state->b_from_domain[0] == '\0' && len > 5){
|
||||
q = strchr(puf, '@');
|
||||
if(q && strlen(q) > 5){
|
||||
memcpy(&(state->b_from_domain), q+1, strlen(q+1)-1);
|
||||
if(strstr(sdata->mailfrom, "<>")){
|
||||
snprintf(sdata->fromemail, SMALLBUFSIZE-1, "%s", puf);
|
||||
sdata->fromemail[len-1] = '\0';
|
||||
}
|
||||
}
|
||||
|
||||
if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1;
|
||||
|
||||
if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
|
||||
split_email_address(puf);
|
||||
memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
|
||||
strtolower(puf);
|
||||
|
||||
if(state->message_state == MSG_RECIPIENT && is_string_on_list(state->journal_recipient, puf) == 0){
|
||||
append_list(&(state->journal_recipient), puf);
|
||||
memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
|
||||
memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
|
||||
}
|
||||
|
||||
|
||||
if(is_string_on_list(state->rcpt, puf) == 0){
|
||||
append_list(&(state->rcpt), puf);
|
||||
memcpy(&(state->b_to[state->tolen]), puf, len);
|
||||
state->tolen += len;
|
||||
|
||||
if(does_it_seem_like_an_email_address(puf) == 1){
|
||||
if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_recipient = 1;
|
||||
else sdata->external_recipient = 1;
|
||||
|
||||
q = strchr(puf, '@');
|
||||
if(q){
|
||||
if(is_string_on_list(state->rcpt_domain, q+1) == 0){
|
||||
append_list(&(state->rcpt_domain), q+1);
|
||||
memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1));
|
||||
}
|
||||
}
|
||||
|
||||
if(state->tolen < MAXBUFSIZE-len-1){
|
||||
split_email_address(puf);
|
||||
memcpy(&(state->b_to[state->tolen]), puf, len);
|
||||
state->tolen += len;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
else if(state->message_state == MSG_BODY && len >= cfg->min_word_len && state->bodylen < BIGBUFSIZE-len-1){
|
||||
memcpy(&(state->b_body[state->bodylen]), puf, len);
|
||||
state->bodylen += len;
|
||||
}
|
||||
|
||||
} while(p);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
39
src/parser.h
39
src/parser.h
@ -9,35 +9,30 @@
|
||||
#include "config.h"
|
||||
#include "defs.h"
|
||||
|
||||
struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg);
|
||||
void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg);
|
||||
int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg);
|
||||
struct _state parse_message(struct session_data *sdata, int take_into_pieces, struct __data *data, struct __config *cfg);
|
||||
void post_parse(struct session_data *sdata, struct _state *state, struct __config *cfg);
|
||||
int parse_line(char *buf, struct _state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct __data *data, struct __config *cfg);
|
||||
|
||||
void init_state(struct parser_state *state);
|
||||
time_t parse_date_header(char *s);
|
||||
int extract_boundary(char *p, struct parser_state *state);
|
||||
void fixupEncodedHeaderLine(char *buf, int buflen);
|
||||
void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state);
|
||||
void fixupBase64EncodedLine(char *buf, struct parser_state *state);
|
||||
void markHTML(char *buf, struct parser_state *state);
|
||||
void setStateHTML(char *htmlbuf, int pos, struct parser_state *state);
|
||||
void translateLine(unsigned char *p, struct parser_state *state);
|
||||
void init_state(struct _state *state);
|
||||
unsigned long parse_date_header(char *s, struct __config *cfg);
|
||||
int isHexNumber(char *p);
|
||||
int extract_boundary(char *p, struct _state *state);
|
||||
void fixupEncodedHeaderLine(char *buf);
|
||||
void fixupSoftBreakInQuotedPritableLine(char *buf, struct _state *state);
|
||||
void fixupBase64EncodedLine(char *buf, struct _state *state);
|
||||
void markHTML(char *buf, struct _state *state);
|
||||
int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct _state *state);
|
||||
void translateLine(unsigned char *p, struct _state *state);
|
||||
void fix_email_address_for_sphinx(char *s);
|
||||
void split_email_address(char *s);
|
||||
int does_it_seem_like_an_email_address(char *email);
|
||||
void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
|
||||
void reassembleToken(char *p);
|
||||
void degenerateToken(unsigned char *p);
|
||||
void fixURL(char *buf, int buflen);
|
||||
void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen);
|
||||
void fixURL(char *url);
|
||||
int extractNameFromHeaderLine(char *s, char *name, char *resultbuf);
|
||||
char *determine_attachment_type(char *filename, char *type);
|
||||
char *get_attachment_extractor_by_filename(char *filename);
|
||||
void parse_reference(struct parser_state *state, char *s);
|
||||
int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen);
|
||||
void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len);
|
||||
void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, struct data *data, struct config *cfg);
|
||||
void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize);
|
||||
void fill_attachment_name_buf(struct parser_state *state, char *buf);
|
||||
int get_first_email_address_from_string(char *str, char *buf, int buflen);
|
||||
void parse_reference(struct _state *state, char *s);
|
||||
int base64_decode_attachment_buffer(char *p, int plen, unsigned char *b, int blen);
|
||||
|
||||
#endif /* _PARSER_H */
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
353
src/piler-smtp.c
353
src/piler-smtp.c
@ -1,353 +0,0 @@
|
||||
/*
|
||||
* piler-smtp.c
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <netdb.h>
|
||||
#include <pwd.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <signal.h>
|
||||
#include <sys/epoll.h>
|
||||
#include <netinet/in.h>
|
||||
#include <locale.h>
|
||||
#include <errno.h>
|
||||
#include <sys/time.h>
|
||||
#include <time.h>
|
||||
#include <syslog.h>
|
||||
#include <openssl/ssl.h>
|
||||
#include <openssl/err.h>
|
||||
#include <piler.h>
|
||||
|
||||
#define PROGNAME "piler-smtp"
|
||||
|
||||
extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
struct epoll_event event, *events=NULL;
|
||||
int num_connections = 0;
|
||||
int listenerfd = -1;
|
||||
int loglevel = 1;
|
||||
|
||||
char *configfile = CONFIG_FILE;
|
||||
struct config cfg;
|
||||
struct passwd *pwd;
|
||||
struct smtp_session *session, **sessions=NULL;
|
||||
struct smtp_acl *smtp_acl[MAXHASH];
|
||||
|
||||
time_t prev_timeout_check = 0;
|
||||
|
||||
void usage(){
|
||||
printf("\nusage: piler\n\n");
|
||||
printf(" -c <config file> Config file to use if not the default\n");
|
||||
printf(" -d Fork to the background\n");
|
||||
printf(" -v Return the version and build number\n");
|
||||
printf(" -V Return the version and some build parameters\n");
|
||||
printf(" -L <log level> Set the log level: 1-5\n");
|
||||
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
void p_clean_exit(int sig){
|
||||
if(sig > 0) syslog(LOG_PRIORITY, "got signal: %d, %s", sig, strsignal(sig));
|
||||
|
||||
if(listenerfd != -1) close(listenerfd);
|
||||
|
||||
if(sessions){
|
||||
for(int i=0; i<cfg.max_connections; i++){
|
||||
if(sessions[i]) free_smtp_session(sessions[i]);
|
||||
}
|
||||
|
||||
free(sessions);
|
||||
}
|
||||
|
||||
if(events) free(events);
|
||||
|
||||
clear_smtp_acl(smtp_acl);
|
||||
|
||||
syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
|
||||
|
||||
ERR_free_strings();
|
||||
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
||||
void fatal(char *s){
|
||||
syslog(LOG_PRIORITY, "%s", s);
|
||||
p_clean_exit(0);
|
||||
}
|
||||
|
||||
|
||||
void check_for_client_timeout(){
|
||||
time_t now;
|
||||
|
||||
time(&now);
|
||||
|
||||
if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now);
|
||||
|
||||
if(now - prev_timeout_check < cfg.smtp_timeout) return;
|
||||
|
||||
if(num_connections > 0){
|
||||
for(int i=0; i<cfg.max_connections; i++){
|
||||
if(sessions[i] && now - sessions[i]->lasttime >= cfg.smtp_timeout){
|
||||
syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime);
|
||||
tear_down_session(sessions, sessions[i]->slot, &num_connections, "timeout");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
time(&prev_timeout_check);
|
||||
}
|
||||
|
||||
|
||||
void initialise_configuration(){
|
||||
cfg = read_config(configfile);
|
||||
|
||||
if(strlen(cfg.username) > 1){
|
||||
pwd = getpwnam(cfg.username);
|
||||
if(!pwd) fatal(ERR_NON_EXISTENT_USER);
|
||||
}
|
||||
|
||||
if(getuid() == 0 && pwd){
|
||||
check_and_create_directories(&cfg, pwd->pw_uid, pwd->pw_gid);
|
||||
}
|
||||
|
||||
if(chdir(cfg.workdir)){
|
||||
syslog(LOG_PRIORITY, "workdir: *%s*", cfg.workdir);
|
||||
fatal(ERR_CHDIR);
|
||||
}
|
||||
|
||||
setlocale(LC_MESSAGES, cfg.locale);
|
||||
setlocale(LC_CTYPE, cfg.locale);
|
||||
|
||||
load_smtp_acl(smtp_acl);
|
||||
|
||||
syslog(LOG_PRIORITY, "reloaded config: %s", configfile);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int client_sockfd;
|
||||
int i, daemonise=0;
|
||||
int client_len = sizeof(struct sockaddr_storage);
|
||||
ssize_t readlen;
|
||||
struct sockaddr_storage client_address;
|
||||
char readbuf[REALLYBIGBUFSIZE];
|
||||
int efd;
|
||||
|
||||
while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
|
||||
switch(i){
|
||||
|
||||
case 'c' :
|
||||
configfile = optarg;
|
||||
break;
|
||||
|
||||
case 'L':
|
||||
loglevel = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'd' :
|
||||
daemonise = 1;
|
||||
break;
|
||||
|
||||
case 'v' :
|
||||
case 'V' :
|
||||
printf("%s build %d\n", VERSION, get_build());
|
||||
return 0;
|
||||
|
||||
case 'h' :
|
||||
default :
|
||||
usage();
|
||||
}
|
||||
}
|
||||
|
||||
(void) openlog(PROGNAME, LOG_PID, LOG_MAIL);
|
||||
|
||||
initialise_configuration();
|
||||
|
||||
listenerfd = create_and_bind(cfg.listen_addr, cfg.listen_port);
|
||||
if(listenerfd == -1){
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if(make_socket_non_blocking(listenerfd) == -1){
|
||||
fatal("make_socket_non_blocking()");
|
||||
}
|
||||
|
||||
if(listen(listenerfd, cfg.backlog) == -1){
|
||||
fatal("ERROR: listen()");
|
||||
}
|
||||
|
||||
if(drop_privileges(pwd)) fatal(ERR_SETUID);
|
||||
|
||||
efd = epoll_create1(0);
|
||||
if(efd == -1){
|
||||
fatal("ERROR: epoll_create()");
|
||||
}
|
||||
|
||||
event.data.fd = listenerfd;
|
||||
event.events = EPOLLIN | EPOLLET;
|
||||
if(epoll_ctl(efd, EPOLL_CTL_ADD, listenerfd, &event) == -1){
|
||||
fatal("ERROR: epoll_ctl() on efd");
|
||||
}
|
||||
|
||||
set_signal_handler(SIGINT, p_clean_exit);
|
||||
set_signal_handler(SIGTERM, p_clean_exit);
|
||||
set_signal_handler(SIGKILL, p_clean_exit);
|
||||
set_signal_handler(SIGSEGV, p_clean_exit);
|
||||
|
||||
set_signal_handler(SIGPIPE, SIG_IGN);
|
||||
set_signal_handler(SIGALRM, SIG_IGN);
|
||||
|
||||
set_signal_handler(SIGHUP, initialise_configuration);
|
||||
|
||||
// calloc() initialitizes the allocated memory
|
||||
|
||||
sessions = calloc(cfg.max_connections, sizeof(struct smtp_session));
|
||||
events = calloc(cfg.max_connections, sizeof(struct epoll_event));
|
||||
|
||||
if(!sessions || !events) fatal("ERROR: calloc()");
|
||||
|
||||
SSL_library_init();
|
||||
SSL_load_error_strings();
|
||||
|
||||
srand(getpid());
|
||||
|
||||
syslog(LOG_PRIORITY, "%s %s, build %d starting", PROGNAME, VERSION, get_build());
|
||||
|
||||
#if HAVE_DAEMON == 1
|
||||
if(daemonise == 1 && daemon(1, 0) == -1) fatal(ERR_DAEMON);
|
||||
#endif
|
||||
|
||||
for(;;){
|
||||
int n = epoll_wait(efd, events, cfg.max_connections, 1000);
|
||||
for(i=0; i<n; i++){
|
||||
|
||||
// Office365 sometimes behaves oddly: when it receives the 250 OK
|
||||
// message after sending the email, it doesn't send the QUIT command
|
||||
// rather it aborts the connection
|
||||
|
||||
if((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))){
|
||||
if(cfg.verbosity >= _LOG_EXTREME) syslog(LOG_PRIORITY, "ERROR: the remote end hung up without sending QUIT");
|
||||
session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
|
||||
if(session)
|
||||
tear_down_session(sessions, session->slot, &num_connections, "hungup");
|
||||
else
|
||||
close(events[i].data.fd);
|
||||
continue;
|
||||
}
|
||||
|
||||
// We have 1 or more incoming connections to process
|
||||
|
||||
else if(listenerfd == events[i].data.fd){
|
||||
|
||||
while(1){
|
||||
|
||||
client_sockfd = accept(listenerfd, (struct sockaddr *)&client_address, (socklen_t *)&client_len);
|
||||
if(client_sockfd == -1){
|
||||
if((errno == EAGAIN) || (errno == EWOULDBLOCK)){
|
||||
// We have processed all incoming connections
|
||||
break;
|
||||
}
|
||||
else {
|
||||
syslog(LOG_PRIORITY, "ERROR: accept(): '%s'", strerror(errno));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
|
||||
memset(hbuf, 0, sizeof(hbuf));
|
||||
memset(sbuf, 0, sizeof(sbuf));
|
||||
|
||||
if(getnameinfo((struct sockaddr *)&client_address, client_len, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
|
||||
// Strictly speaking it's not correct to log num_connections+1 connections
|
||||
// but it still gives a good clue how many connections we have at the moment
|
||||
syslog(LOG_PRIORITY, "connected from %s:%s on fd=%d (active connections: %d)", hbuf, sbuf, client_sockfd, num_connections + 1);
|
||||
}
|
||||
|
||||
if(make_socket_non_blocking(client_sockfd) == -1){
|
||||
syslog(LOG_PRIORITY, "ERROR: cannot make the socket non blocking");
|
||||
break;
|
||||
}
|
||||
|
||||
event.data.fd = client_sockfd;
|
||||
event.events = EPOLLIN | EPOLLET;
|
||||
if(epoll_ctl(efd, EPOLL_CTL_ADD, client_sockfd, &event) == -1){
|
||||
syslog(LOG_PRIORITY, "ERROR: epoll_ctl() on client_sockfd");
|
||||
break;
|
||||
}
|
||||
|
||||
start_new_session(sessions, client_sockfd, &num_connections, smtp_acl, hbuf, &cfg);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// handle data from an existing connection
|
||||
|
||||
else {
|
||||
int done = 0;
|
||||
|
||||
session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
|
||||
if(session == NULL){
|
||||
syslog(LOG_PRIORITY, "ERROR: cannot find session for this socket: %d", events[i].data.fd);
|
||||
close(events[i].data.fd);
|
||||
continue;
|
||||
}
|
||||
|
||||
time(&(session->lasttime));
|
||||
|
||||
while(1){
|
||||
if(session->net.use_ssl == 1)
|
||||
readlen = SSL_read(session->net.ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
|
||||
else
|
||||
readlen = read(events[i].data.fd, (char*)&readbuf[0], sizeof(readbuf)-1);
|
||||
|
||||
if(cfg.verbosity >= _LOG_EXTREME && readlen > 0) syslog(LOG_PRIORITY, "got %ld bytes to read", readlen);
|
||||
|
||||
if(readlen == -1){
|
||||
/* If errno == EAGAIN, that means we have read all data. So go back to the main loop. */
|
||||
if(errno != EAGAIN){
|
||||
done = 1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
else if(readlen == 0){
|
||||
/* End of file. The remote has closed the connection. */
|
||||
done = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
readbuf[readlen] = '\0';
|
||||
handle_data(session, &readbuf[0], readlen, &cfg);
|
||||
|
||||
if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){
|
||||
done = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Don't wait until the remote client closes the connection after he sent the QUIT command */
|
||||
|
||||
if(done || session->protocol_state == SMTP_STATE_FINISHED){
|
||||
tear_down_session(sessions, session->slot, &num_connections, "done");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
check_for_client_timeout();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
415
src/piler.c
415
src/piler.c
@ -8,7 +8,6 @@
|
||||
#include <strings.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/mman.h>
|
||||
#include <netdb.h>
|
||||
#include <netinet/in.h>
|
||||
#include <arpa/inet.h>
|
||||
@ -21,41 +20,59 @@
|
||||
#include <syslog.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
#include <locale.h>
|
||||
#include <errno.h>
|
||||
#include <openssl/ssl.h>
|
||||
#include <openssl/err.h>
|
||||
#include <piler.h>
|
||||
|
||||
#define PROGNAME "piler"
|
||||
|
||||
extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
int sd;
|
||||
int quit = 0;
|
||||
int received_sighup = 0;
|
||||
char *configfile = CONFIG_FILE;
|
||||
struct config cfg;
|
||||
struct data data;
|
||||
struct __config cfg;
|
||||
struct __data data;
|
||||
struct passwd *pwd;
|
||||
|
||||
struct child children[MAXCHILDREN];
|
||||
|
||||
|
||||
signal_func *set_signal_handler(int signo, signal_func * func);
|
||||
static void takesig(int sig);
|
||||
static void child_sighup_handler(int sig);
|
||||
static void child_main(struct child *ptr);
|
||||
static pid_t child_make(struct child *ptr);
|
||||
int search_slot_by_pid(pid_t pid);
|
||||
void kill_children(int sig);
|
||||
void p_clean_exit();
|
||||
void fatal(char *s);
|
||||
void initialise_configuration();
|
||||
|
||||
|
||||
void usage(){
|
||||
printf("\nusage: piler\n\n");
|
||||
printf(" -c <config file> Config file to use if not the default\n");
|
||||
printf(" -d Fork to the background\n");
|
||||
printf(" -v Return the version and build number\n");
|
||||
printf(" -V Return the version and some build parameters\n");
|
||||
|
||||
exit(0);
|
||||
|
||||
/*
|
||||
* most of the preforking code was taken from the tinyproxy project
|
||||
*/
|
||||
|
||||
signal_func *set_signal_handler(int signo, signal_func * func){
|
||||
struct sigaction act, oact;
|
||||
|
||||
act.sa_handler = func;
|
||||
sigemptyset (&act.sa_mask);
|
||||
act.sa_flags = 0;
|
||||
|
||||
if(sigaction(signo, &act, &oact) < 0) return SIG_ERR;
|
||||
|
||||
return oact.sa_handler;
|
||||
}
|
||||
|
||||
|
||||
void takesig(int sig){
|
||||
static void takesig(int sig){
|
||||
int i, status;
|
||||
pid_t pid;
|
||||
|
||||
@ -67,6 +84,7 @@ void takesig(int sig){
|
||||
break;
|
||||
|
||||
case SIGTERM:
|
||||
case SIGKILL:
|
||||
quit = 1;
|
||||
p_clean_exit();
|
||||
break;
|
||||
@ -79,7 +97,6 @@ void takesig(int sig){
|
||||
if(quit == 0){
|
||||
i = search_slot_by_pid(pid);
|
||||
if(i >= 0){
|
||||
children[i].serial = i;
|
||||
children[i].status = READY;
|
||||
children[i].pid = child_make(&children[i]);
|
||||
}
|
||||
@ -94,190 +111,28 @@ void takesig(int sig){
|
||||
}
|
||||
|
||||
|
||||
void child_sighup_handler(int sig){
|
||||
static void child_sighup_handler(int sig){
|
||||
if(sig == SIGHUP){
|
||||
received_sighup = 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){
|
||||
|
||||
if(cfg->security_header[0] && parser_state->found_security_header == 0){
|
||||
syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename);
|
||||
return ERR_DISCARDED;
|
||||
}
|
||||
|
||||
char *arule = check_against_ruleset(data->archiving_rules, parser_state, sdata->tot_len, sdata->spam_message);
|
||||
|
||||
if(arule){
|
||||
syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
|
||||
return ERR_DISCARDED;
|
||||
}
|
||||
|
||||
|
||||
if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){
|
||||
syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename);
|
||||
return ERR_DISCARDED;
|
||||
}
|
||||
|
||||
make_digests(sdata, cfg);
|
||||
|
||||
// A normal header is much bigger than 10 bytes. We get here for header-only
|
||||
// messages without a Message-ID: line. I believe that no such message is valid, and
|
||||
// it's a reasonable to discard it, and not allowing it to fill up the error directory.
|
||||
|
||||
if(sdata->hdr_len < 10){
|
||||
syslog(LOG_PRIORITY, "%s: discarding: a header-only message without a Message-ID line", filename);
|
||||
return ERR_DISCARDED;
|
||||
}
|
||||
|
||||
int rc = process_message(sdata, parser_state, data, cfg);
|
||||
unlink(parser_state->message_id_hash);
|
||||
|
||||
return rc;
|
||||
void *get_in_addr(struct sockaddr *sa){
|
||||
if(sa->sa_family == AF_INET) return &(((struct sockaddr_in*)sa)->sin_addr);
|
||||
return &(((struct sockaddr_in6*)sa)->sin6_addr);
|
||||
}
|
||||
|
||||
|
||||
int process_email(char *filename, struct session_data *sdata, struct data *data, int size, struct config *cfg){
|
||||
char tmpbuf[SMALLBUFSIZE];
|
||||
char *status=S_STATUS_UNDEF;
|
||||
char *p;
|
||||
struct timezone tz;
|
||||
struct timeval tv1, tv2;
|
||||
struct parser_state parser_state;
|
||||
struct counters counters;
|
||||
|
||||
gettimeofday(&tv1, &tz);
|
||||
|
||||
bzero(&counters, sizeof(counters));
|
||||
|
||||
#ifdef HAVE_ANTIVIRUS
|
||||
if(do_av_check(filename, cfg) == AVIR_VIRUS){
|
||||
unlink(filename);
|
||||
return OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
init_session_data(sdata, cfg);
|
||||
|
||||
sdata->tot_len = size;
|
||||
|
||||
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
|
||||
|
||||
parser_state = parse_message(sdata, 1, data, cfg);
|
||||
|
||||
post_parse(sdata, &parser_state, cfg);
|
||||
|
||||
if(cfg->syslog_recipients == 1){
|
||||
char *rcpt = parser_state.b_to;
|
||||
do {
|
||||
rcpt = split_str(rcpt, " ", tmpbuf, sizeof(tmpbuf)-1);
|
||||
|
||||
if(does_it_seem_like_an_email_address(tmpbuf) == 1){
|
||||
syslog(LOG_PRIORITY, "%s: rcpt=%s", sdata->ttmpfile, tmpbuf);
|
||||
}
|
||||
} while(rcpt);
|
||||
}
|
||||
|
||||
int rc = perform_checks(filename, sdata, data, &parser_state, cfg);
|
||||
unlink(sdata->tmpframe);
|
||||
|
||||
remove_stripped_attachments(&parser_state);
|
||||
|
||||
if(rc == OK){
|
||||
status = S_STATUS_STORED;
|
||||
counters.c_rcvd = 1;
|
||||
counters.c_size += sdata->tot_len;
|
||||
counters.c_stored_size = sdata->stored_len;
|
||||
}
|
||||
else if(rc == ERR_EXISTS){
|
||||
status = S_STATUS_DUPLICATE;
|
||||
counters.c_duplicate = 1;
|
||||
syslog(LOG_PRIORITY, "%s: discarding: duplicate message, id: %llu, message-id: %s", filename, sdata->duplicate_id, parser_state.message_id);
|
||||
}
|
||||
else if(rc == ERR_DISCARDED){
|
||||
status = S_STATUS_DISCARDED;
|
||||
counters.c_ignore = 1;
|
||||
}
|
||||
else {
|
||||
status = S_STATUS_ERROR;
|
||||
|
||||
// move the file from piler/tmp/[0-xxx] dir to piler/error directory
|
||||
p = strchr(filename, '/');
|
||||
if(p)
|
||||
p++;
|
||||
else
|
||||
p = filename;
|
||||
|
||||
snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s/%s", ERROR_DIR, p);
|
||||
if(rename(filename, tmpbuf) == 0)
|
||||
syslog(LOG_PRIORITY, "%s: moved to %s", filename, tmpbuf);
|
||||
else
|
||||
syslog(LOG_PRIORITY, "%s: failed to moved to %s", filename, tmpbuf);
|
||||
}
|
||||
|
||||
if(rc != ERR) unlink(filename);
|
||||
|
||||
update_counters(sdata, data, &counters, cfg);
|
||||
|
||||
gettimeofday(&tv2, &tz);
|
||||
|
||||
syslog(LOG_PRIORITY, "%s: %s, size=%d/%d, attachments=%d, reference=%s, "
|
||||
"message-id=%s, retention=%d, folder=%d, delay=%.4f, status=%s",
|
||||
filename, sdata->ttmpfile, sdata->tot_len, sdata->stored_len,
|
||||
parser_state.n_attachments, parser_state.reference, parser_state.message_id,
|
||||
parser_state.retention, data->folder, tvdiff(tv2,tv1)/1000000.0, status);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int process_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){
|
||||
DIR *dir;
|
||||
struct dirent *de;
|
||||
int tot_msgs=0;
|
||||
char fname[SMALLBUFSIZE];
|
||||
struct stat st;
|
||||
|
||||
dir = opendir(directory);
|
||||
if(!dir){
|
||||
syslog(LOG_PRIORITY, "cannot open directory: %s", directory);
|
||||
return tot_msgs;
|
||||
}
|
||||
|
||||
while((de = readdir(dir))){
|
||||
if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
|
||||
|
||||
snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
|
||||
|
||||
if(stat(fname, &st) == 0){
|
||||
if(S_ISREG(st.st_mode) && process_email(fname, sdata, data, st.st_size, cfg) != ERR){
|
||||
tot_msgs++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
syslog(LOG_PRIORITY, "ERROR: cannot stat: %s", fname);
|
||||
}
|
||||
}
|
||||
|
||||
closedir(dir);
|
||||
|
||||
return tot_msgs;
|
||||
}
|
||||
|
||||
|
||||
void child_main(struct child *ptr){
|
||||
struct session_data sdata;
|
||||
char dir[TINYBUFSIZE];
|
||||
|
||||
/* open directory, then process its files, then sleep 1 sec, and repeat */
|
||||
static void child_main(struct child *ptr){
|
||||
int new_sd;
|
||||
char s[INET6_ADDRSTRLEN];
|
||||
struct sockaddr_storage client_addr;
|
||||
socklen_t addr_size;
|
||||
|
||||
ptr->messages = 0;
|
||||
|
||||
snprintf(dir, sizeof(dir)-1, "%d", ptr->serial);
|
||||
|
||||
if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) started main() working on '%s'", getpid(), ptr->serial, dir);
|
||||
if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d) started main()", getpid());
|
||||
|
||||
while(1){
|
||||
if(received_sighup == 1){
|
||||
@ -285,39 +140,35 @@ void child_main(struct child *ptr){
|
||||
break;
|
||||
}
|
||||
|
||||
ptr->status = READY;
|
||||
|
||||
addr_size = sizeof(client_addr);
|
||||
new_sd = accept(sd, (struct sockaddr *)&client_addr, &addr_size);
|
||||
|
||||
if(new_sd == -1) continue;
|
||||
|
||||
ptr->status = BUSY;
|
||||
|
||||
inet_ntop(client_addr.ss_family, get_in_addr((struct sockaddr *)&client_addr), s, sizeof(s));
|
||||
|
||||
syslog(LOG_PRIORITY, "connection from %s", s);
|
||||
|
||||
|
||||
sig_block(SIGHUP);
|
||||
|
||||
int sphxopen = 0;
|
||||
if(cfg.rtindex && open_sphx(&sdata, &cfg) == OK){
|
||||
sphxopen = 1;
|
||||
}
|
||||
|
||||
if((cfg.rtindex == 0 || sphxopen == 1) && open_database(&sdata, &cfg) == OK){
|
||||
ptr->messages += process_dir(dir, &sdata, &data, &cfg);
|
||||
close_database(&sdata);
|
||||
|
||||
if(cfg.rtindex){
|
||||
close_sphx(&sdata);
|
||||
}
|
||||
|
||||
sleep(1);
|
||||
}
|
||||
else {
|
||||
syslog(LOG_PRIORITY, "ERROR: cannot open database");
|
||||
sleep(10);
|
||||
}
|
||||
|
||||
ptr->messages += handle_smtp_session(new_sd, &data, &cfg);
|
||||
sig_unblock(SIGHUP);
|
||||
|
||||
// Let the child quit after processing max_requests_per_child messages
|
||||
close(new_sd);
|
||||
|
||||
if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
|
||||
if(cfg.verbosity >= _LOG_DEBUG)
|
||||
syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
|
||||
if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d) served enough: %d", getpid(), ptr->messages);
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ptr->status = UNDEF;
|
||||
|
||||
#ifdef HAVE_MEMCACHED
|
||||
memcached_shutdown(&(data.memc));
|
||||
#endif
|
||||
@ -328,7 +179,7 @@ void child_main(struct child *ptr){
|
||||
}
|
||||
|
||||
|
||||
pid_t child_make(struct child *ptr){
|
||||
static pid_t child_make(struct child *ptr){
|
||||
pid_t pid;
|
||||
|
||||
if((pid = fork()) > 0) return pid;
|
||||
@ -357,12 +208,10 @@ int child_pool_create(){
|
||||
children[i].pid = 0;
|
||||
children[i].messages = 0;
|
||||
children[i].status = UNDEF;
|
||||
children[i].serial = -1;
|
||||
}
|
||||
|
||||
for(i=0; i<cfg.number_of_worker_processes; i++){
|
||||
children[i].status = READY;
|
||||
children[i].serial = i;
|
||||
children[i].pid = child_make(&children[i]);
|
||||
|
||||
if(children[i].pid == -1){
|
||||
@ -399,19 +248,23 @@ void kill_children(int sig){
|
||||
|
||||
|
||||
void p_clean_exit(){
|
||||
if(sd != -1) close(sd);
|
||||
|
||||
kill_children(SIGTERM);
|
||||
|
||||
clearrules(data.archiving_rules);
|
||||
clearrules(data.retention_rules);
|
||||
clearrules(data.folder_rules);
|
||||
|
||||
clearhash(data.mydomains);
|
||||
free_rule(data.archiving_rules);
|
||||
free_rule(data.retention_rules);
|
||||
|
||||
syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
|
||||
|
||||
unlink(cfg.pidfile);
|
||||
|
||||
if(data.dedup != MAP_FAILED) munmap(data.dedup, MAXCHILDREN*DIGEST_LENGTH*2);
|
||||
#ifdef HAVE_STARTTLS
|
||||
if(data.ctx){
|
||||
SSL_CTX_free(data.ctx);
|
||||
ERR_free_strings();
|
||||
}
|
||||
#endif
|
||||
|
||||
exit(1);
|
||||
}
|
||||
@ -423,12 +276,33 @@ void fatal(char *s){
|
||||
}
|
||||
|
||||
|
||||
#ifdef HAVE_STARTTLS
|
||||
int init_ssl(){
|
||||
|
||||
SSL_library_init();
|
||||
SSL_load_error_strings();
|
||||
|
||||
data.ctx = SSL_CTX_new(SSLv23_server_method());
|
||||
|
||||
if(data.ctx == NULL){ syslog(LOG_PRIORITY, "SSL_CTX_new() failed"); return ERR; }
|
||||
|
||||
if(SSL_CTX_set_cipher_list(data.ctx, cfg.cipher_list) == 0){ syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", cfg.cipher_list); return ERR; }
|
||||
|
||||
if(SSL_CTX_use_PrivateKey_file(data.ctx, cfg.pemfile, SSL_FILETYPE_PEM) != 1){ syslog(LOG_PRIORITY, "cannot load private key from %s", cfg.pemfile); return ERR; }
|
||||
|
||||
if(SSL_CTX_use_certificate_file(data.ctx, cfg.pemfile, SSL_FILETYPE_PEM) != 1){ syslog(LOG_PRIORITY, "cannot load certificate from %s", cfg.pemfile); return ERR; }
|
||||
|
||||
return OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
void initialise_configuration(){
|
||||
struct session_data sdata;
|
||||
|
||||
cfg = read_config(configfile);
|
||||
|
||||
if(cfg.number_of_worker_processes < 2) cfg.number_of_worker_processes = 2;
|
||||
if(cfg.number_of_worker_processes < 5) cfg.number_of_worker_processes = 5;
|
||||
if(cfg.number_of_worker_processes > MAXCHILDREN) cfg.number_of_worker_processes = MAXCHILDREN;
|
||||
|
||||
if(strlen(cfg.username) > 1){
|
||||
@ -451,34 +325,37 @@ void initialise_configuration(){
|
||||
setlocale(LC_CTYPE, cfg.locale);
|
||||
|
||||
|
||||
clearrules(data.archiving_rules);
|
||||
clearrules(data.retention_rules);
|
||||
clearrules(data.folder_rules);
|
||||
|
||||
clearhash(data.mydomains);
|
||||
free_rule(data.archiving_rules);
|
||||
free_rule(data.retention_rules);
|
||||
|
||||
data.folder = 0;
|
||||
data.recursive_folder_names = 0;
|
||||
data.archiving_rules = NULL;
|
||||
data.retention_rules = NULL;
|
||||
|
||||
inithash(data.mydomains);
|
||||
initrules(data.archiving_rules);
|
||||
initrules(data.retention_rules);
|
||||
initrules(data.folder_rules);
|
||||
memset(data.starttls, 0, sizeof(data.starttls));
|
||||
|
||||
if(open_database(&sdata, &cfg) == ERR){
|
||||
#ifdef HAVE_STARTTLS
|
||||
if(cfg.tls_enable > 0 && data.ctx == NULL && init_ssl() == OK){
|
||||
snprintf(data.starttls, sizeof(data.starttls)-1, "250-STARTTLS\r\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
mysql_init(&(sdata.mysql));
|
||||
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
|
||||
if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
|
||||
syslog(LOG_PRIORITY, "cannot connect to mysql server");
|
||||
return;
|
||||
}
|
||||
|
||||
load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
|
||||
load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
|
||||
load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
|
||||
load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
|
||||
load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
|
||||
|
||||
load_mydomains(&sdata, &data, &cfg);
|
||||
|
||||
if(cfg.server_id > 0) insert_offset(&sdata, cfg.server_id);
|
||||
|
||||
close_database(&sdata);
|
||||
mysql_close(&(sdata.mysql));
|
||||
|
||||
|
||||
syslog(LOG_PRIORITY, "reloaded config: %s", configfile);
|
||||
@ -490,8 +367,9 @@ void initialise_configuration(){
|
||||
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int i, daemonise=0;
|
||||
struct stat st;
|
||||
int i, rc, yes=1, daemonise=0;
|
||||
char port_string[6];
|
||||
struct addrinfo hints, *res;
|
||||
|
||||
|
||||
while((i = getopt(argc, argv, "c:dvVh")) > 0){
|
||||
@ -506,17 +384,13 @@ int main(int argc, char **argv){
|
||||
break;
|
||||
|
||||
case 'v' :
|
||||
printf("%s build %d\n", VERSION, get_build());
|
||||
return 0;
|
||||
|
||||
case 'V' :
|
||||
printf("%s %s, build %d, Janos SUTO <sj@acts.hu>\n\n%s\nMySQL client library version: %s\n", PROGNAME, VERSION, get_build(), CONFIGURE_PARAMS, mysql_get_client_info());
|
||||
get_extractor_list();
|
||||
printf("%s %s, build %d, Janos SUTO <sj@acts.hu>\n\n%s\n\n", PROGNAME, VERSION, get_build(), CONFIGURE_PARAMS);
|
||||
return 0;
|
||||
|
||||
case 'h' :
|
||||
default :
|
||||
usage();
|
||||
__fatal("usage: ...");
|
||||
}
|
||||
}
|
||||
|
||||
@ -524,12 +398,11 @@ int main(int argc, char **argv){
|
||||
|
||||
data.folder = 0;
|
||||
data.recursive_folder_names = 0;
|
||||
inithash(data.mydomains);
|
||||
initrules(data.archiving_rules);
|
||||
initrules(data.retention_rules);
|
||||
initrules(data.folder_rules);
|
||||
data.dedup = MAP_FAILED;
|
||||
data.import = NULL;
|
||||
data.archiving_rules = NULL;
|
||||
data.retention_rules = NULL;
|
||||
data.ctx = NULL;
|
||||
data.ssl = NULL;
|
||||
|
||||
|
||||
initialise_configuration();
|
||||
|
||||
@ -539,37 +412,59 @@ int main(int argc, char **argv){
|
||||
if(read_key(&cfg)) fatal(ERR_READING_KEY);
|
||||
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
snprintf(port_string, sizeof(port_string)-1, "%d", cfg.listen_port);
|
||||
|
||||
if((rc = getaddrinfo(cfg.listen_addr, port_string, &hints, &res)) != 0){
|
||||
fprintf(stderr, "getaddrinfo for '%s': %s\n", cfg.listen_addr, gai_strerror(rc));
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1)
|
||||
fatal(ERR_OPEN_SOCKET);
|
||||
|
||||
if(setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1)
|
||||
fatal(ERR_SET_SOCK_OPT);
|
||||
|
||||
if(bind(sd, res->ai_addr, res->ai_addrlen) == -1)
|
||||
fatal(ERR_BIND_TO_PORT);
|
||||
|
||||
if(listen(sd, cfg.backlog) == -1)
|
||||
fatal(ERR_LISTEN);
|
||||
|
||||
|
||||
freeaddrinfo(res);
|
||||
|
||||
|
||||
if(drop_privileges(pwd)) fatal(ERR_SETUID);
|
||||
|
||||
if(stat(cfg.pidfile, &st) == 0) fatal(ERR_PID_FILE_EXISTS);
|
||||
|
||||
if(cfg.mmap_dedup_test == 1){
|
||||
int dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
|
||||
if(dedupfd == -1) fatal(ERR_OPEN_DEDUP_FILE);
|
||||
|
||||
data.dedup = mmap(NULL, MAXCHILDREN*DIGEST_LENGTH*2, PROT_READ|PROT_WRITE, MAP_SHARED, dedupfd, 0);
|
||||
close(dedupfd);
|
||||
|
||||
if(data.dedup == MAP_FAILED) syslog(LOG_INFO, "cannot mmap() %s, errno=%d", MESSAGE_ID_DEDUP_FILE, errno);
|
||||
}
|
||||
|
||||
syslog(LOG_PRIORITY, "%s %s, build %d starting", PROGNAME, VERSION, get_build());
|
||||
|
||||
|
||||
#if HAVE_DAEMON == 1
|
||||
if(daemonise == 1 && daemon(1, 0) == -1) fatal(ERR_DAEMON);
|
||||
if(daemonise == 1) i = daemon(1, 0);
|
||||
#endif
|
||||
|
||||
write_pid_file(cfg.pidfile);
|
||||
|
||||
|
||||
child_pool_create();
|
||||
|
||||
set_signal_handler(SIGCHLD, takesig);
|
||||
set_signal_handler(SIGTERM, takesig);
|
||||
set_signal_handler(SIGKILL, takesig);
|
||||
set_signal_handler(SIGHUP, takesig);
|
||||
|
||||
|
||||
for(;;){ sleep(1); }
|
||||
|
||||
p_clean_exit();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
75
src/piler.h
75
src/piler.h
@ -6,21 +6,18 @@
|
||||
#define _PILER_H
|
||||
|
||||
#include <misc.h>
|
||||
#include <list.h>
|
||||
#include <parser.h>
|
||||
#include <errmsg.h>
|
||||
#include <smtpcodes.h>
|
||||
#include <decoder.h>
|
||||
#include <hash.h>
|
||||
#include <list.h>
|
||||
#include <rules.h>
|
||||
#include <defs.h>
|
||||
#include <tai.h>
|
||||
#include <sig.h>
|
||||
#include <av.h>
|
||||
#include <rules.h>
|
||||
#include <screen.h>
|
||||
#include <sql.h>
|
||||
#include <import.h>
|
||||
#include <smtp.h>
|
||||
#include <config.h>
|
||||
#include <unistd.h>
|
||||
|
||||
@ -28,62 +25,48 @@
|
||||
#include "memc.h"
|
||||
#endif
|
||||
|
||||
int read_key(struct config *cfg);
|
||||
int read_key(struct __config *cfg);
|
||||
void insert_offset(struct session_data *sdata, int server_id);
|
||||
|
||||
void tear_down_client(int n);
|
||||
int do_av_check(struct session_data *sdata, char *rcpttoemail, char *virusinfo, struct __data *data, struct __config *cfg);
|
||||
|
||||
int do_av_check(char *filename, struct config *cfg);
|
||||
|
||||
int make_digests(struct session_data *sdata, struct config *cfg);
|
||||
int make_digests(struct session_data *sdata, struct __config *cfg);
|
||||
void digest_file(char *filename, char *digest);
|
||||
void digest_string(char *digestname, char *s, char *digest);
|
||||
void create_md5_from_email_address(char *puf, char *md5buf);
|
||||
void digest_string(char *s, char *digest);
|
||||
|
||||
void remove_stripped_attachments(struct parser_state *state);
|
||||
int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
|
||||
void rollback(struct session_data *sdata, struct parser_state *state, uint64 id, struct config *cfg);
|
||||
int reimport_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
|
||||
int store_file(struct session_data *sdata, char *filename, int len, struct config *cfg);
|
||||
int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct config *cfg);
|
||||
int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg);
|
||||
int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr);
|
||||
int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg);
|
||||
|
||||
struct config read_config(char *configfile);
|
||||
void remove_stripped_attachments(struct _state *state);
|
||||
int process_message(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg);
|
||||
int store_file(struct session_data *sdata, char *filename, int startpos, int len, struct __config *cfg);
|
||||
int remove_stored_message_files(struct session_data *sdata, struct _state *state, struct __config *cfg);
|
||||
int store_attachments(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg);
|
||||
int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr, struct __config *cfg);
|
||||
|
||||
void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
|
||||
void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid);
|
||||
struct __config read_config(char *configfile);
|
||||
|
||||
void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
|
||||
void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid);
|
||||
|
||||
int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg);
|
||||
int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct data *data, struct config *cfg);
|
||||
void update_counters(struct session_data *sdata, struct __data *data, struct __counters *counters, struct __config *cfg);
|
||||
|
||||
int get_folder_id(struct session_data *sdata, char *foldername, int parent_id);
|
||||
int add_new_folder(struct session_data *sdata, char *foldername, int parent_id);
|
||||
int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg);
|
||||
|
||||
int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg);
|
||||
int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s);
|
||||
|
||||
void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg);
|
||||
int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg);
|
||||
unsigned long get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
|
||||
unsigned long add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
|
||||
|
||||
int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg);
|
||||
int store_index_data(struct session_data *sdata, struct _state *state, struct __data *data, uint64 id, struct __config *cfg);
|
||||
|
||||
void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg);
|
||||
int is_email_address_on_my_domains(char *email, struct data *data);
|
||||
void extract_attachment_content(struct session_data *sdata, struct _state *state, char *filename, char *type, int *rec);
|
||||
|
||||
int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg);
|
||||
void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason);
|
||||
struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket);
|
||||
void write_envelope_addresses(struct smtp_session *session, struct config *cfg);
|
||||
void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
|
||||
void free_smtp_session(struct smtp_session *session);
|
||||
int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg);
|
||||
|
||||
void child_sighup_handler(int sig);
|
||||
void child_main(struct child *ptr);
|
||||
pid_t child_make(struct child *ptr);
|
||||
int search_slot_by_pid(pid_t pid);
|
||||
void kill_children(int sig);
|
||||
void fatal(char *s);
|
||||
void initialise_configuration();
|
||||
void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg);
|
||||
|
||||
int create_prepared_statements(struct session_data *sdata, struct __data *data);
|
||||
void close_prepared_statements(struct __data *data);
|
||||
|
||||
#endif /* _PILER_H */
|
||||
|
||||
|
||||
@ -19,10 +19,8 @@
|
||||
int main(int argc, char **argv){
|
||||
int readkey=1;
|
||||
char filename[SMALLBUFSIZE];
|
||||
#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
|
||||
struct stat st;
|
||||
#endif
|
||||
struct config cfg;
|
||||
struct __config cfg;
|
||||
|
||||
|
||||
if(argc < 3){
|
||||
@ -46,14 +44,14 @@ int main(int argc, char **argv){
|
||||
return 1;
|
||||
}
|
||||
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, argv[1][24], argv[1][25], argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
|
||||
#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
|
||||
if(stat(filename, &st)){
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
|
||||
}
|
||||
#endif
|
||||
|
||||
retrieve_file_from_archive(filename, WRITE_TO_STDOUT, NULL, stdout, &cfg);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -12,13 +12,13 @@ extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
|
||||
void print_config_all(struct config *cfg, char *key);
|
||||
void print_config(char *configfile, struct config *cfg);
|
||||
void print_config_all(struct __config *cfg, char *key);
|
||||
void print_config(char *configfile, struct __config *cfg);
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int i, print_from_file=0;
|
||||
char *configfile=CONFIG_FILE, *query=NULL;
|
||||
struct config cfg;
|
||||
struct __config cfg;
|
||||
|
||||
while((i = getopt(argc, argv, "c:q:nh?")) > 0){
|
||||
switch(i){
|
||||
@ -41,7 +41,7 @@ int main(int argc, char **argv){
|
||||
break;
|
||||
|
||||
|
||||
default :
|
||||
default :
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,13 +8,11 @@
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/mman.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <syslog.h>
|
||||
#include <zip.h>
|
||||
#include <getopt.h>
|
||||
#include <piler.h>
|
||||
|
||||
@ -23,48 +21,19 @@ extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
int dryrun = 0;
|
||||
int exportall = 0;
|
||||
int verification_status = 0;
|
||||
int export_to_stdout = 0;
|
||||
char *query=NULL;
|
||||
int verbosity = 0;
|
||||
int max_matches = 1000;
|
||||
char *index_list = "main1,dailydelta1,delta1";
|
||||
struct passwd *pwd;
|
||||
regex_t regexp;
|
||||
char *zipfile = NULL;
|
||||
struct zip *zip = NULL;
|
||||
uint64 *zip_ids = NULL;
|
||||
int zip_counter = 0;
|
||||
int zip_batch = 2000;
|
||||
int max_files_in_export_dir = 0;
|
||||
|
||||
int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg);
|
||||
|
||||
|
||||
void usage(){
|
||||
printf("\nusage: pilerexport\n\n");
|
||||
|
||||
printf(" [-c <config file>] Config file to use if not the default\n");
|
||||
printf(" -a <start date> Start date in YYYY.MM.DD format\n");
|
||||
printf(" -b <stop date> Stop date in YYYY.MM.DD format\n");
|
||||
printf(" -f <email@address.com> From address\n");
|
||||
printf(" -r <email@address.com> Recipient address\n");
|
||||
printf(" -F <domain.com> From domain\n");
|
||||
printf(" -R <domain.com> Recipient domain\n");
|
||||
printf(" -s <size> Min. size\n");
|
||||
printf(" -S <size> Max. size\n");
|
||||
printf(" -w <where condition> Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n");
|
||||
printf(" -m <max. matches> Max. matches to apply to sphinx query (default: %d)\n", max_matches);
|
||||
printf(" -i <index list> Sphinx indices to use (default: %s)\n", index_list);
|
||||
#if LIBZIP_VERSION_MAJOR >= 1
|
||||
printf(" -z <zip file> Write exported EML files to a zip file\n");
|
||||
printf(" -Z <batch size> Zip batch size. Valid range: 10-10000, default: 2000\n");
|
||||
#endif
|
||||
printf(" -A Export all emails from archive\n");
|
||||
printf(" -D <max files> Max. number of files to put in a single directory, default: 2000\n");
|
||||
printf(" -o Export emails to stdout\n");
|
||||
printf(" -d Dry run\n");
|
||||
printf("\nusage: pilerexport \n\n");
|
||||
printf(" [-c|--config <config file>] \n");
|
||||
printf(" -a|--start-date <YYYY.MM.DD> -b|--stop-date <YYYY.MM.DD> \n");
|
||||
printf(" -f|--from <email@address> -r|--to <email@address>\n");
|
||||
printf(" -s|--minsize <number> -S|--maxsize <number>\n");
|
||||
printf(" -A|--all -d|--dryrun \n");
|
||||
printf("\n use -A if you don't want to specify the start/stop time nor any from/to address\n\n");
|
||||
|
||||
regfree(®exp);
|
||||
|
||||
@ -102,12 +71,15 @@ unsigned long convert_time(char *yyyymmdd, int h, int m, int s){
|
||||
|
||||
tm.tm_mday = atoi(yyyymmdd);
|
||||
|
||||
|
||||
tm.tm_isdst = -1;
|
||||
|
||||
return mktime(&tm);
|
||||
}
|
||||
|
||||
|
||||
int append_email_to_buffer(char **buffer, char *email){
|
||||
int arglen;
|
||||
int len, arglen;
|
||||
char *s=NULL, emailaddress[SMALLBUFSIZE];
|
||||
|
||||
snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email);
|
||||
@ -119,7 +91,7 @@ int append_email_to_buffer(char **buffer, char *email){
|
||||
memcpy(*buffer, emailaddress, arglen);
|
||||
}
|
||||
else {
|
||||
int len = strlen(*buffer);
|
||||
len = strlen(*buffer);
|
||||
s = realloc(*buffer, len + arglen+2);
|
||||
if(!s){
|
||||
printf("malloc problem!\n");
|
||||
@ -137,326 +109,88 @@ int append_email_to_buffer(char **buffer, char *email){
|
||||
}
|
||||
|
||||
|
||||
uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){
|
||||
int append_string_to_buffer(char **buffer, char *str){
|
||||
int len, arglen;
|
||||
char *s=NULL;
|
||||
|
||||
arglen = strlen(str);
|
||||
|
||||
if(!*buffer){
|
||||
*buffer = malloc(arglen+1);
|
||||
memset(*buffer, 0, arglen+1);
|
||||
memcpy(*buffer, str, arglen);
|
||||
}
|
||||
else {
|
||||
len = strlen(*buffer);
|
||||
s = realloc(*buffer, len + arglen+1);
|
||||
if(!s) return 1;
|
||||
|
||||
*buffer = s;
|
||||
|
||||
memset(*buffer+len, 0, arglen+1);
|
||||
memcpy(*buffer+len, str, arglen);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg){
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
uint64 id=0;
|
||||
char s[MAXBUFSIZE];
|
||||
int rc=0;
|
||||
|
||||
*num = 0;
|
||||
|
||||
if(!where_condition) return id;
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);
|
||||
|
||||
if(dryrun){
|
||||
printf("sphinx query: %s\n", s);
|
||||
}
|
||||
|
||||
syslog(LOG_PRIORITY, "sphinx query: %s", s);
|
||||
|
||||
if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
|
||||
MYSQL_RES *res = mysql_store_result(&(sdata2->mysql));
|
||||
if(res != NULL){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
id = strtoull(row[0], NULL, 10);
|
||||
(*num)++;
|
||||
rc += append_string_to_buffer(&query, row[0]);
|
||||
rc += append_string_to_buffer(&query, ",");
|
||||
|
||||
}
|
||||
mysql_free_result(res);
|
||||
|
||||
rc += append_string_to_buffer(&query, "-1)");
|
||||
}
|
||||
}
|
||||
|
||||
if(!rc) export_emails_matching_to_query(sdata, query, cfg);
|
||||
else printf("error: append_string_to_buffer() in run_query()\n");
|
||||
|
||||
free(query);
|
||||
query = NULL;
|
||||
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
uint64 get_total_found(struct session_data *sdata){
|
||||
MYSQL_ROW row;
|
||||
uint64 total_found=0;
|
||||
|
||||
if(mysql_real_query(&(sdata->mysql), "SHOW META LIKE 'total_found'", 28) == 0){
|
||||
MYSQL_RES *res = mysql_store_result(&(sdata->mysql));
|
||||
if(res != NULL){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
total_found = strtoull(row[1], NULL, 10);
|
||||
}
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
return total_found;
|
||||
}
|
||||
|
||||
|
||||
void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, char *where_condition, struct config *cfg){
|
||||
int n;
|
||||
uint64 count=0, last_id=0, total_found=0;
|
||||
|
||||
last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
|
||||
count += n;
|
||||
|
||||
total_found = get_total_found(sdata2);
|
||||
|
||||
while(count < total_found){
|
||||
last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
|
||||
count += n;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
|
||||
char s[SMALLBUFSIZE];
|
||||
int rc=0;
|
||||
|
||||
if(exportall == 1){
|
||||
rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM ");
|
||||
rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
|
||||
rc += append_string_to_buffer(&query, " WHERE deleted=0 ");
|
||||
return rc;
|
||||
}
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
|
||||
|
||||
rc = append_string_to_buffer(&query, s);
|
||||
|
||||
if(from){
|
||||
rc = append_string_to_buffer(&query, " AND ");
|
||||
|
||||
rc += append_string_to_buffer(&query, "`from` IN (");
|
||||
rc += append_string_to_buffer(&query, from);
|
||||
rc += append_string_to_buffer(&query, ")");
|
||||
|
||||
free(from);
|
||||
}
|
||||
|
||||
if(to){
|
||||
rc = append_string_to_buffer(&query, " AND ");
|
||||
|
||||
rc += append_string_to_buffer(&query, "`to` IN (");
|
||||
rc += append_string_to_buffer(&query, to);
|
||||
rc += append_string_to_buffer(&query, ")");
|
||||
|
||||
free(to);
|
||||
}
|
||||
|
||||
if(fromdomain){
|
||||
rc = append_string_to_buffer(&query, " AND ");
|
||||
|
||||
rc += append_string_to_buffer(&query, "`fromdomain` IN (");
|
||||
rc += append_string_to_buffer(&query, fromdomain);
|
||||
rc += append_string_to_buffer(&query, ")");
|
||||
|
||||
free(fromdomain);
|
||||
}
|
||||
|
||||
|
||||
if(todomain){
|
||||
rc = append_string_to_buffer(&query, " AND ");
|
||||
|
||||
rc += append_string_to_buffer(&query, "`todomain` IN (");
|
||||
rc += append_string_to_buffer(&query, todomain);
|
||||
rc += append_string_to_buffer(&query, ")");
|
||||
|
||||
free(todomain);
|
||||
}
|
||||
|
||||
if(minsize > 0){
|
||||
rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
}
|
||||
|
||||
|
||||
if(maxsize > 0){
|
||||
rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
}
|
||||
|
||||
|
||||
if(startdate > 0){
|
||||
rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
}
|
||||
|
||||
|
||||
if(stopdate > 0){
|
||||
rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
}
|
||||
|
||||
|
||||
rc += append_string_to_buffer(&query, " ORDER BY id ASC");
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
#if LIBZIP_VERSION_MAJOR >= 1
|
||||
void zip_flush(){
|
||||
zip_close(zip);
|
||||
|
||||
zip = NULL;
|
||||
zip_counter = 0;
|
||||
|
||||
if(!zip_ids) return;
|
||||
|
||||
for(int i=0; i<zip_batch; i++){
|
||||
if(*(zip_ids+i)){
|
||||
char filename[SMALLBUFSIZE];
|
||||
snprintf(filename, sizeof(filename)-1, "%llu.eml", *(zip_ids+i));
|
||||
unlink(filename);
|
||||
}
|
||||
}
|
||||
|
||||
free(zip_ids);
|
||||
zip_ids = NULL;
|
||||
}
|
||||
#endif
|
||||
|
||||
int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){
|
||||
FILE *f;
|
||||
uint64 id, n=0, dir_counter=0;
|
||||
char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
|
||||
uint64 id, n=0;
|
||||
char *digest=NULL, *bodydigest=NULL;
|
||||
char filename[SMALLBUFSIZE];
|
||||
char export_subdir[SMALLBUFSIZE];
|
||||
struct sql sql;
|
||||
int errorp, rc=0, attachments;
|
||||
unsigned long total_attachments=0;
|
||||
|
||||
if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR;
|
||||
int rc=0;
|
||||
|
||||
|
||||
p_bind_init(&sql);
|
||||
rc = mysql_real_query(&(sdata->mysql), s, strlen(s));
|
||||
|
||||
if(p_exec_stmt(sdata, &sql) == ERR) goto ENDE;
|
||||
if(rc == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
|
||||
id = strtoull(row[0], NULL, 10);
|
||||
if(id > 0){
|
||||
snprintf(sdata->ttmpfile, SMALLBUFSIZE-1, "%s", (char*)row[1]);
|
||||
digest = (char*)row[2];
|
||||
bodydigest = (char*)row[3];
|
||||
|
||||
if(dryrun == 0){
|
||||
|
||||
p_bind_init(&sql);
|
||||
snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
|
||||
|
||||
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
|
||||
sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
|
||||
sql.sql[sql.pos] = &digest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(digest)-2; sql.pos++;
|
||||
sql.sql[sql.pos] = &bodydigest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(bodydigest)-2; sql.pos++;
|
||||
sql.sql[sql.pos] = (char *)&attachments; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
|
||||
f = fopen(filename, "w");
|
||||
if(f){
|
||||
rc = retrieve_email_from_archive(sdata, data, f, cfg);
|
||||
fclose(f);
|
||||
|
||||
p_store_results(&sql);
|
||||
n++;
|
||||
|
||||
while(p_fetch_results(&sql) == OK){
|
||||
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
|
||||
|
||||
if(id > 0){
|
||||
make_digests(sdata, cfg);
|
||||
|
||||
if(dryrun == 0){
|
||||
if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
|
||||
printf("exported: %10llu\r", n); fflush(stdout);
|
||||
}
|
||||
else
|
||||
printf("verification FAILED. %s\n", filename);
|
||||
|
||||
if(export_to_stdout){
|
||||
printf("%s", PILEREXPORT_BEGIN_MARK);
|
||||
rc = retrieve_email_from_archive(sdata, stdout, cfg);
|
||||
continue;
|
||||
}
|
||||
|
||||
if(max_files_in_export_dir > 0 && n % max_files_in_export_dir == 0){
|
||||
dir_counter++;
|
||||
snprintf(export_subdir, sizeof(export_subdir)-1, "export-%llu", dir_counter);
|
||||
if(n > 0 && chdir("..")){
|
||||
p_clean_exit("error chdir(\"..\")", 1);
|
||||
}
|
||||
|
||||
createdir(export_subdir, pwd->pw_uid, pwd->pw_gid, 0700);
|
||||
if(chdir(export_subdir)){
|
||||
p_clean_exit("error chdir to export-* dir", 1);
|
||||
}
|
||||
}
|
||||
|
||||
snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
|
||||
|
||||
f = fopen(filename, "w");
|
||||
if(f){
|
||||
rc = retrieve_email_from_archive(sdata, f, cfg);
|
||||
fclose(f);
|
||||
|
||||
n++;
|
||||
|
||||
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
|
||||
|
||||
make_digests(sdata, cfg);
|
||||
|
||||
if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
|
||||
printf("exported: %10llu\r", n); fflush(stdout);
|
||||
}
|
||||
else printf("cannot open: %s\n", filename);
|
||||
}
|
||||
else {
|
||||
printf("verification FAILED. %s\n", filename);
|
||||
verification_status = 1;
|
||||
printf("id:%llu\n", id);
|
||||
}
|
||||
|
||||
if(zipfile){
|
||||
#if LIBZIP_VERSION_MAJOR >= 1
|
||||
// Open zip file if handler is NULL
|
||||
if(!zip){
|
||||
zip = zip_open(zipfile, ZIP_CREATE, &errorp);
|
||||
if(!zip){
|
||||
printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
|
||||
return ERR;
|
||||
}
|
||||
}
|
||||
|
||||
if(!zip_ids) zip_ids = (uint64*) calloc(sizeof(uint64), zip_batch);
|
||||
|
||||
if(!zip_ids){
|
||||
printf("calloc error for zip_ids\n");
|
||||
return ERR;
|
||||
}
|
||||
|
||||
zip_source_t *zs = zip_source_file(zip, filename, 0, 0);
|
||||
if(zs && zip_file_add(zip, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
|
||||
*(zip_ids+zip_counter) = id;
|
||||
zip_counter++;
|
||||
} else {
|
||||
printf("error adding file %s: %s\n", filename, zip_strerror(zip));
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(zip_counter == zip_batch){
|
||||
zip_flush();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else printf("cannot open: %s\n", filename);
|
||||
}
|
||||
else {
|
||||
total_attachments += attachments;
|
||||
printf("id:%llu\n", id);
|
||||
}
|
||||
|
||||
mysql_free_result(res);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
p_free_results(&sql);
|
||||
|
||||
ENDE:
|
||||
close_prepared_statement(&sql);
|
||||
|
||||
if(dryrun){
|
||||
printf("attachments: %lu\n", total_attachments);
|
||||
else rc = 1;
|
||||
}
|
||||
|
||||
printf("\n");
|
||||
@ -466,13 +200,16 @@ ENDE:
|
||||
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int minsize=0, maxsize=0;
|
||||
int c, rc, exportall=0, minsize=0, maxsize=0;
|
||||
int where_condition=0;
|
||||
size_t nmatch=0;
|
||||
unsigned long startdate=0, stopdate=0;
|
||||
char *configfile=CONFIG_FILE;
|
||||
char *to=NULL, *from=NULL, *todomain=NULL, *fromdomain=NULL, *where_condition=NULL;
|
||||
struct session_data sdata, sdata2;
|
||||
struct config cfg;
|
||||
char *to=NULL, *from=NULL;
|
||||
char s[SMALLBUFSIZE];
|
||||
struct session_data sdata;
|
||||
struct __data data;
|
||||
struct __config cfg;
|
||||
|
||||
|
||||
if(regcomp(®exp, "^([\\+a-z0-9_\\.@\\-]+)$", REG_ICASE | REG_EXTENDED)){
|
||||
@ -491,29 +228,21 @@ int main(int argc, char **argv){
|
||||
{"all", no_argument, 0, 'A' },
|
||||
{"dry-run", no_argument, 0, 'd' },
|
||||
{"dryrun", no_argument, 0, 'd' },
|
||||
{"stdout", no_argument, 0, 'o' },
|
||||
{"help", no_argument, 0, 'h' },
|
||||
{"version", no_argument, 0, 'v' },
|
||||
{"from", required_argument, 0, 'f' },
|
||||
{"to", required_argument, 0, 'r' },
|
||||
{"from-domain", required_argument, 0, 'F' },
|
||||
{"to-domain", required_argument, 0, 'R' },
|
||||
{"start-date", required_argument, 0, 'a' },
|
||||
{"stop-date", required_argument, 0, 'b' },
|
||||
{"zip", required_argument, 0, 'z' },
|
||||
{"zip-batch", required_argument, 0, 'Z' },
|
||||
{"where-condition", required_argument, 0, 'w' },
|
||||
{"max-files", required_argument, 0, 'D' },
|
||||
{"max-matches", required_argument, 0, 'm' },
|
||||
{"index-list", required_argument, 0, 'i' },
|
||||
{"id", required_argument, 0, 'i' },
|
||||
{0,0,0,0}
|
||||
};
|
||||
|
||||
int option_index = 0;
|
||||
|
||||
int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?", long_options, &option_index);
|
||||
c = getopt_long(argc, argv, "c:s:S:f:r:a:b:i:Adhv?", long_options, &option_index);
|
||||
#else
|
||||
int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?");
|
||||
c = getopt(argc, argv, "c:s:S:f:r:a:b:i:Adhv?");
|
||||
#endif
|
||||
|
||||
if(c == -1) break;
|
||||
@ -544,10 +273,7 @@ int main(int argc, char **argv){
|
||||
break;
|
||||
}
|
||||
|
||||
if(append_email_to_buffer(&from, optarg)){
|
||||
printf("error: append_email_to_buffer() for from\n");
|
||||
return 1;
|
||||
}
|
||||
rc = append_email_to_buffer(&from, optarg);
|
||||
|
||||
break;
|
||||
|
||||
@ -558,40 +284,10 @@ int main(int argc, char **argv){
|
||||
break;
|
||||
}
|
||||
|
||||
if(append_email_to_buffer(&to, optarg)){
|
||||
printf("error: append_email_to_buffer() for to\n");
|
||||
return 1;
|
||||
}
|
||||
rc = append_email_to_buffer(&to, optarg);
|
||||
|
||||
break;
|
||||
|
||||
case 'F' :
|
||||
|
||||
if(regexec(®exp, optarg, nmatch, NULL, 0)){
|
||||
printf("%s is not a valid domain name\n", optarg);
|
||||
break;
|
||||
}
|
||||
|
||||
if(append_email_to_buffer(&fromdomain, optarg)){
|
||||
printf("error: append_email_to_buffer() for fromdomain\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case 'R' :
|
||||
|
||||
if(regexec(®exp, optarg, nmatch, NULL, 0)){
|
||||
printf("%s is not a valid domain name\n", optarg);
|
||||
break;
|
||||
}
|
||||
|
||||
if(append_email_to_buffer(&todomain, optarg)){
|
||||
printf("error: append_email_to_buffer() for todomain\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case 'a' :
|
||||
startdate = convert_time(optarg, 0, 0, 0);
|
||||
@ -602,39 +298,12 @@ int main(int argc, char **argv){
|
||||
stopdate = convert_time(optarg, 23, 59, 59);
|
||||
break;
|
||||
|
||||
case 'w' :
|
||||
where_condition = optarg;
|
||||
break;
|
||||
|
||||
case 'm' :
|
||||
max_matches = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'i' :
|
||||
index_list = optarg;
|
||||
break;
|
||||
|
||||
case 'z': zipfile = optarg;
|
||||
break;
|
||||
|
||||
case 'Z': zip_batch = atoi(optarg);
|
||||
if(zip_batch < 10 || zip_batch > 10000)
|
||||
zip_batch = 2000;
|
||||
break;
|
||||
|
||||
case 'D': max_files_in_export_dir = atoi(optarg);
|
||||
if(max_files_in_export_dir < 10 || max_files_in_export_dir > 100000)
|
||||
max_files_in_export_dir = 2000;
|
||||
break;
|
||||
|
||||
case 'o':
|
||||
export_to_stdout = 1;
|
||||
break;
|
||||
|
||||
case 'd' :
|
||||
dryrun = 1;
|
||||
break;
|
||||
|
||||
|
||||
default :
|
||||
usage();
|
||||
break;
|
||||
@ -643,64 +312,117 @@ int main(int argc, char **argv){
|
||||
}
|
||||
|
||||
|
||||
if(from == NULL && to == NULL && fromdomain == NULL && todomain == NULL && where_condition == NULL && startdate == 0 && stopdate == 0 && exportall == 0) usage();
|
||||
if(from == NULL && to == NULL && startdate == 0 && stopdate == 0 && exportall == 0) usage();
|
||||
|
||||
|
||||
regfree(®exp);
|
||||
|
||||
if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
|
||||
|
||||
(void) openlog("pilerexport", LOG_PID, LOG_MAIL);
|
||||
|
||||
if(exportall == 1){
|
||||
rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM ");
|
||||
rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
|
||||
goto GO;
|
||||
}
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE ", SQL_MESSAGES_VIEW);
|
||||
|
||||
rc = append_string_to_buffer(&query, s);
|
||||
|
||||
if(from){
|
||||
rc += append_string_to_buffer(&query, "`from` IN (");
|
||||
rc += append_string_to_buffer(&query, from);
|
||||
rc += append_string_to_buffer(&query, ")");
|
||||
|
||||
free(from);
|
||||
|
||||
where_condition++;
|
||||
}
|
||||
|
||||
|
||||
if(to){
|
||||
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
|
||||
|
||||
rc += append_string_to_buffer(&query, "`to` IN (");
|
||||
rc += append_string_to_buffer(&query, to);
|
||||
rc += append_string_to_buffer(&query, ")");
|
||||
|
||||
free(to);
|
||||
|
||||
where_condition++;
|
||||
}
|
||||
|
||||
|
||||
if(minsize > 0){
|
||||
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
|
||||
where_condition++;
|
||||
}
|
||||
|
||||
|
||||
if(maxsize > 0){
|
||||
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
|
||||
where_condition++;
|
||||
}
|
||||
|
||||
|
||||
if(startdate > 0){
|
||||
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `sent` >= %ld", startdate);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
|
||||
where_condition++;
|
||||
}
|
||||
|
||||
|
||||
if(stopdate > 0){
|
||||
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `sent` <= %ld", stopdate);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
|
||||
where_condition++;
|
||||
}
|
||||
|
||||
|
||||
rc += append_string_to_buffer(&query, " ORDER BY id ASC");
|
||||
|
||||
|
||||
GO:
|
||||
if(rc) p_clean_exit("malloc problem building query", 1);
|
||||
|
||||
|
||||
cfg = read_config(configfile);
|
||||
|
||||
|
||||
if(read_key(&cfg)) p_clean_exit(ERR_READING_KEY, 1);
|
||||
|
||||
if(strlen(cfg.username) > 1){
|
||||
pwd = getpwnam(cfg.username);
|
||||
if(!pwd) __fatal(ERR_NON_EXISTENT_USER);
|
||||
}
|
||||
|
||||
|
||||
init_session_data(&sdata, &cfg);
|
||||
|
||||
|
||||
if(open_database(&sdata, &cfg) == ERR){
|
||||
mysql_init(&(sdata.mysql));
|
||||
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
|
||||
if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
|
||||
p_clean_exit("cannot connect to mysql server", 1);
|
||||
}
|
||||
|
||||
mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
|
||||
mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
|
||||
|
||||
if(where_condition){
|
||||
|
||||
init_session_data(&sdata2, &cfg);
|
||||
rc = export_emails_matching_to_query(&sdata, &data, query, &cfg);
|
||||
|
||||
strcpy(cfg.mysqlhost, "127.0.0.1");
|
||||
cfg.mysqlport = 9306;
|
||||
cfg.mysqlsocket[0] = '\0';
|
||||
free(query);
|
||||
|
||||
if(open_database(&sdata2, &cfg) == ERR){
|
||||
p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
|
||||
}
|
||||
|
||||
export_emails_matching_id_list(&sdata, &sdata2, where_condition, &cfg);
|
||||
mysql_close(&(sdata.mysql));
|
||||
|
||||
close_database(&sdata2);
|
||||
}
|
||||
else {
|
||||
if(build_query_from_args(from, to, fromdomain, todomain, minsize, maxsize, startdate, stopdate) > 0) p_clean_exit("malloc problem building query", 1);
|
||||
export_emails_matching_to_query(&sdata, query, &cfg);
|
||||
free(query);
|
||||
}
|
||||
|
||||
close_database(&sdata);
|
||||
|
||||
if(zipfile){
|
||||
#if LIBZIP_VERSION_MAJOR >= 1
|
||||
zip_flush();
|
||||
#endif
|
||||
}
|
||||
|
||||
return verification_status;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -19,7 +19,8 @@
|
||||
int main(int argc, char **argv){
|
||||
int readkey=1;
|
||||
struct session_data sdata;
|
||||
struct config cfg;
|
||||
struct __data data;
|
||||
struct __config cfg;
|
||||
|
||||
|
||||
if(argc < 2){
|
||||
@ -31,10 +32,7 @@ int main(int argc, char **argv){
|
||||
|
||||
cfg = read_config(CONFIG_FILE);
|
||||
|
||||
if(argc >= 3){
|
||||
readkey = 0;
|
||||
cfg.encrypt_messages = 0;
|
||||
}
|
||||
if(argc >= 3) readkey = 0;
|
||||
|
||||
if(readkey == 1 && read_key(&cfg)){
|
||||
printf("%s\n", ERR_READING_KEY);
|
||||
@ -42,15 +40,25 @@ int main(int argc, char **argv){
|
||||
}
|
||||
|
||||
|
||||
if(open_database(&sdata, &cfg) == ERR) return 0;
|
||||
mysql_init(&(sdata.mysql));
|
||||
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
|
||||
if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
|
||||
printf("cannot connect to mysql server\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
|
||||
mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
|
||||
|
||||
|
||||
snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
|
||||
snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile);
|
||||
retrieve_email_from_archive(&sdata, stdout, &cfg);
|
||||
retrieve_email_from_archive(&sdata, &data, stdout, &cfg);
|
||||
|
||||
|
||||
close_database(&sdata);
|
||||
mysql_close(&(sdata.mysql));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -30,97 +30,375 @@
|
||||
extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
int quiet=0;
|
||||
int remove_after_successful_import = 0;
|
||||
|
||||
|
||||
int connect_to_imap_server(int sd, int *seq, char *username, char *password, int port, struct __data *data, int use_ssl);
|
||||
int list_folders(int sd, int *seq, char *folders, int foldersize, int use_ssl, struct __data *data);
|
||||
int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg);
|
||||
int connect_to_pop3_server(int sd, char *username, char *password, int port, struct __data *data, int use_ssl);
|
||||
int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg);
|
||||
void close_connection(int sd, struct __data *data, int use_ssl);
|
||||
|
||||
|
||||
int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg){
|
||||
FILE *F, *f=NULL;
|
||||
int rc=ERR, tot_msgs=0, ret=OK;
|
||||
char buf[MAXBUFSIZE], fname[SMALLBUFSIZE];
|
||||
time_t t;
|
||||
|
||||
|
||||
F = fopen(mailbox, "r");
|
||||
if(!F){
|
||||
printf("cannot open mailbox: %s\n", mailbox);
|
||||
return rc;
|
||||
}
|
||||
|
||||
t = time(NULL);
|
||||
|
||||
while(fgets(buf, sizeof(buf)-1, F)){
|
||||
|
||||
if(buf[0] == 'F' && buf[1] == 'r' && buf[2] == 'o' && buf[3] == 'm' && buf[4] == ' '){
|
||||
tot_msgs++;
|
||||
if(f){
|
||||
fclose(f);
|
||||
f = NULL;
|
||||
rc = import_message(fname, sdata, data, cfg);
|
||||
if(rc == ERR){
|
||||
printf("error importing: '%s'\n", fname);
|
||||
ret = ERR;
|
||||
}
|
||||
else unlink(fname);
|
||||
|
||||
if(quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
|
||||
}
|
||||
|
||||
snprintf(fname, sizeof(fname)-1, "%ld-%d", t, tot_msgs);
|
||||
f = fopen(fname, "w+");
|
||||
continue;
|
||||
}
|
||||
|
||||
if(f) fprintf(f, "%s", buf);
|
||||
}
|
||||
|
||||
if(f){
|
||||
fclose(f);
|
||||
rc = import_message(fname, sdata, data, cfg);
|
||||
if(rc == ERR){
|
||||
printf("error importing: '%s'\n", fname);
|
||||
ret = ERR;
|
||||
}
|
||||
else unlink(fname);
|
||||
|
||||
if(quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
|
||||
}
|
||||
|
||||
fclose(F);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
|
||||
DIR *dir;
|
||||
struct dirent *de;
|
||||
int rc=ERR, ret=OK, i=0;
|
||||
unsigned long folder;
|
||||
char fname[SMALLBUFSIZE];
|
||||
struct stat st;
|
||||
|
||||
dir = opendir(directory);
|
||||
if(!dir){
|
||||
printf("cannot open directory: %s\n", directory);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
while((de = readdir(dir))){
|
||||
if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
|
||||
|
||||
snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
|
||||
|
||||
if(stat(fname, &st) == 0){
|
||||
if(S_ISDIR(st.st_mode)){
|
||||
folder = data->folder;
|
||||
rc = import_mbox_from_dir(fname, sdata, data, tot_msgs, cfg);
|
||||
data->folder = folder;
|
||||
if(rc == ERR) ret = ERR;
|
||||
}
|
||||
else {
|
||||
|
||||
if(S_ISREG(st.st_mode)){
|
||||
if(i == 0 && data->recursive_folder_names == 1){
|
||||
folder = get_folder_id(sdata, data, fname, data->folder);
|
||||
if(folder == 0){
|
||||
folder = add_new_folder(sdata, data, fname, data->folder);
|
||||
|
||||
if(folder == 0){
|
||||
printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
|
||||
return ERR;
|
||||
}
|
||||
else {
|
||||
data->folder = folder;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rc = import_from_mailbox(fname, sdata, data, cfg);
|
||||
if(rc == OK) (*tot_msgs)++;
|
||||
else ret = ERR;
|
||||
|
||||
i++;
|
||||
}
|
||||
else {
|
||||
printf("%s is not a file\n", fname);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
else {
|
||||
printf("cannot stat() %s\n", fname);
|
||||
}
|
||||
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
|
||||
DIR *dir;
|
||||
struct dirent *de;
|
||||
int rc=ERR, ret=OK, i=0;
|
||||
unsigned long folder;
|
||||
char *p, fname[SMALLBUFSIZE];
|
||||
struct stat st;
|
||||
|
||||
dir = opendir(directory);
|
||||
if(!dir){
|
||||
printf("cannot open directory: %s\n", directory);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
while((de = readdir(dir))){
|
||||
if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
|
||||
|
||||
snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
|
||||
|
||||
if(stat(fname, &st) == 0){
|
||||
if(S_ISDIR(st.st_mode)){
|
||||
folder = data->folder;
|
||||
rc = import_from_maildir(fname, sdata, data, tot_msgs, cfg);
|
||||
data->folder = folder;
|
||||
if(rc == ERR) ret = ERR;
|
||||
}
|
||||
else {
|
||||
|
||||
if(S_ISREG(st.st_mode)){
|
||||
if(i == 0 && data->recursive_folder_names == 1){
|
||||
p = strrchr(directory, '/');
|
||||
if(p) p++;
|
||||
else {
|
||||
printf("invalid directory name: '%s'\n", directory);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
folder = get_folder_id(sdata, data, p, data->folder);
|
||||
if(folder == 0){
|
||||
folder = add_new_folder(sdata, data, p, data->folder);
|
||||
|
||||
if(folder == 0){
|
||||
printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
|
||||
return ERR;
|
||||
}
|
||||
else {
|
||||
data->folder = folder;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rc = import_message(fname, sdata, data, cfg);
|
||||
if(rc == OK) (*tot_msgs)++;
|
||||
else {
|
||||
printf("error importing: '%s'\n", fname);
|
||||
ret = ERR;
|
||||
}
|
||||
|
||||
if(remove_after_successful_import == 1 && ret != ERR) unlink(fname);
|
||||
|
||||
i++;
|
||||
|
||||
if(quiet == 0) printf("processed: %7d\r", *tot_msgs); fflush(stdout);
|
||||
}
|
||||
else {
|
||||
printf("%s is not a file\n", fname);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
else {
|
||||
printf("cannot stat() %s\n", fname);
|
||||
}
|
||||
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *skiplist, struct __config *cfg){
|
||||
int rc=ERR, ret=OK, sd, seq=1, skipmatch, use_ssl=0;
|
||||
char *p, puf[SMALLBUFSIZE];
|
||||
char muf[SMALLBUFSIZE];
|
||||
char folders[MAXBUFSIZE];
|
||||
char port_string[6];
|
||||
struct addrinfo hints, *res;
|
||||
|
||||
snprintf(port_string, sizeof(port_string)-1, "%d", port);
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
|
||||
printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(port == 993) use_ssl = 1;
|
||||
|
||||
|
||||
if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
|
||||
printf("cannot create socket\n");
|
||||
ret = ERR;
|
||||
goto ENDE_IMAP;
|
||||
}
|
||||
|
||||
if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
|
||||
printf("connect()\n");
|
||||
ret = ERR;
|
||||
goto ENDE_IMAP;
|
||||
}
|
||||
|
||||
if(connect_to_imap_server(sd, &seq, username, password, port, data, use_ssl) == ERR){
|
||||
close(sd);
|
||||
ret = ERR;
|
||||
goto ENDE_IMAP;
|
||||
}
|
||||
|
||||
|
||||
list_folders(sd, &seq, &folders[0], sizeof(folders), use_ssl, data);
|
||||
|
||||
|
||||
p = &folders[0];
|
||||
do {
|
||||
memset(puf, 0, sizeof(puf));
|
||||
p = split(p, '\n', puf, sizeof(puf)-1);
|
||||
|
||||
if(strlen(puf) < 1) continue;
|
||||
|
||||
skipmatch = 0;
|
||||
|
||||
if(skiplist && strlen(skiplist) > 0){
|
||||
snprintf(muf, sizeof(muf)-1, "%s,", puf);
|
||||
if(strstr(skiplist, muf)) skipmatch = 1;
|
||||
}
|
||||
|
||||
if(skipmatch == 1){
|
||||
if(quiet == 0) printf("SKIPPING FOLDER: %s\n", puf);
|
||||
continue;
|
||||
}
|
||||
|
||||
if(quiet == 0) printf("processing folder: %s... ", puf);
|
||||
|
||||
if(process_imap_folder(sd, &seq, puf, sdata, data, use_ssl, cfg) == ERR) ret = ERR;
|
||||
|
||||
} while(p);
|
||||
|
||||
|
||||
close_connection(sd, data, use_ssl);
|
||||
|
||||
ENDE_IMAP:
|
||||
freeaddrinfo(res);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, struct __config *cfg){
|
||||
int rc, ret=OK, sd, use_ssl=0;
|
||||
char port_string[6];
|
||||
struct addrinfo hints, *res;
|
||||
|
||||
snprintf(port_string, sizeof(port_string)-1, "%d", port);
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
|
||||
printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(port == 995) use_ssl = 1;
|
||||
|
||||
if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
|
||||
printf("cannot create socket\n");
|
||||
ret = ERR;
|
||||
goto ENDE_POP3;
|
||||
}
|
||||
|
||||
if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
|
||||
printf("connect()\n");
|
||||
ret = ERR;
|
||||
goto ENDE_POP3;
|
||||
}
|
||||
|
||||
|
||||
if(connect_to_pop3_server(sd, username, password, port, data, use_ssl) == ERR){
|
||||
close(sd);
|
||||
ret = ERR;
|
||||
goto ENDE_POP3;
|
||||
}
|
||||
|
||||
if(process_pop3_emails(sd, sdata, data, use_ssl, cfg) == ERR) ret = ERR;
|
||||
|
||||
close_connection(sd, data, use_ssl);
|
||||
|
||||
ENDE_POP3:
|
||||
freeaddrinfo(res);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
void usage(){
|
||||
printf("\nusage: pilerimport\n\n");
|
||||
printf(" [-c <config file>] Config file to use if not the default\n");
|
||||
printf(" -e <eml file> EML file to import\n");
|
||||
printf(" -m <mailbox file> Mbox file to import\n");
|
||||
printf(" -d <dir> Directory with EML files to import\n");
|
||||
printf(" -i <imap server> IMAP server to connect\n");
|
||||
printf(" -K <pop3 server> POP3 server to connect\n");
|
||||
printf(" -u <username> Username for imap/pop3 import\n");
|
||||
printf(" -p <password> Password for imap/pop3 import\n");
|
||||
printf(" -P <port> Port for imap/pop3 import (default: 143/110\n");
|
||||
printf(" -t <timeout> Timeout in sec for imap/pop3 import\n");
|
||||
printf(" -x <folder1,folder2,....folderN,> Comma separated list of imap folders to skip. Add the trailing comma!\n");
|
||||
printf(" -f <imap folder> IMAP folder name to import\n");
|
||||
printf(" -g <imap folder> Move email after import to this IMAP folder\n");
|
||||
printf(" -F <folder> Piler folder name to assign to this import\n");
|
||||
printf(" -R Assign IMAP folder names as Piler folder names\n");
|
||||
printf(" -b <batch limit> Import only this many emails\n");
|
||||
printf(" -s <start position> Start importing POP3 emails from this position\n");
|
||||
printf(" -j <failed folder> Move failed to import emails to this folder\n");
|
||||
printf(" -a <recipient> Add recipient to the To:/Cc: list\n");
|
||||
printf(" -T <id> Update import table at id=<id>\n");
|
||||
printf(" -Z <ms> Delay Z milliseconds in between emails being imported\n");
|
||||
printf(" -D Dry-run, do not import anything\n");
|
||||
printf(" -y Read pilerexport data from stdin\n");
|
||||
printf(" -o Only download emails for POP3/IMAP import\n");
|
||||
printf(" -r Remove imported emails\n");
|
||||
printf(" -q Quiet mode\n");
|
||||
printf(" -A <timestamp> Import emails sent after this timestamp\n");
|
||||
printf(" -B <timestamp> Import emails sent before this timestamp\n");
|
||||
|
||||
printf("usage: pilerimport [-c <config file>] -e <eml file> | -m <mailbox file> | -d <directory> | -i <imap server> | -K <pop3 server> | -u <imap username> -p <imap password> -P <imap port> [-F <foldername>] [-R] [-r] [-q]\n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int i, n_mbox=0, read_from_pilerexport=0;
|
||||
char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL;
|
||||
char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL;
|
||||
int i, c, rc=0, n_mbox=0, tot_msgs=0, port=143;
|
||||
char *configfile=CONFIG_FILE, *emlfile=NULL, *mboxdir=NULL, *mbox[MBOX_ARGS], *directory=NULL;
|
||||
char *imapserver=NULL, *pop3server=NULL, *username=NULL, *password=NULL, *skiplist=SKIPLIST, *folder=NULL;
|
||||
struct session_data sdata;
|
||||
struct config cfg;
|
||||
struct data data;
|
||||
struct import import;
|
||||
struct net net;
|
||||
struct counters counters;
|
||||
|
||||
bzero(&counters, sizeof(counters));
|
||||
struct __config cfg;
|
||||
struct __data data;
|
||||
|
||||
for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
|
||||
|
||||
srand(getpid());
|
||||
|
||||
data.folder = 0;
|
||||
data.recursive_folder_names = 0;
|
||||
data.quiet = 0;
|
||||
|
||||
|
||||
import.import_job_id = import.total_messages = import.total_size = import.processed_messages = import.batch_processing_limit = 0;
|
||||
import.started = import.updated = import.finished = import.remove_after_import = 0;
|
||||
import.extra_recipient = import.move_folder = import.failed_folder = NULL;
|
||||
import.start_position = 1;
|
||||
import.download_only = 0;
|
||||
import.dryrun = 0;
|
||||
import.port = 143;
|
||||
import.server = NULL;
|
||||
import.username = NULL;
|
||||
import.password = NULL;
|
||||
import.database = NULL;
|
||||
import.skiplist = SKIPLIST;
|
||||
import.folder_imap = NULL;
|
||||
memset(import.filename, 0, SMALLBUFSIZE);
|
||||
import.mboxdir = NULL;
|
||||
import.tot_msgs = 0;
|
||||
import.table_id = 0;
|
||||
import.folder = NULL;
|
||||
import.delay = 0;
|
||||
import.after = 0;
|
||||
import.before = 0;
|
||||
|
||||
data.import = &import;
|
||||
|
||||
net.socket = -1;
|
||||
net.timeout = 30;
|
||||
|
||||
data.net = &net;
|
||||
|
||||
inithash(data.mydomains);
|
||||
initrules(data.archiving_rules);
|
||||
initrules(data.retention_rules);
|
||||
initrules(data.folder_rules);
|
||||
data.archiving_rules = NULL;
|
||||
data.retention_rules = NULL;
|
||||
|
||||
while(1){
|
||||
|
||||
@ -139,32 +417,18 @@ int main(int argc, char **argv){
|
||||
{"port", required_argument, 0, 'P' },
|
||||
{"skiplist", required_argument, 0, 'x' },
|
||||
{"folder", required_argument, 0, 'F' },
|
||||
{"folder_imap", required_argument, 0, 'f' },
|
||||
{"add-recipient",required_argument, 0, 'a' },
|
||||
{"batch-limit", required_argument, 0, 'b' },
|
||||
{"timeout", required_argument, 0, 't' },
|
||||
{"start-position",required_argument, 0, 's' },
|
||||
{"table-id", required_argument, 0, 'T' },
|
||||
{"delay", required_argument, 0, 'Z' },
|
||||
{"quiet", no_argument, 0, 'q' },
|
||||
{"recursive", no_argument, 0, 'R' },
|
||||
{"remove-after-import",no_argument, 0, 'r' },
|
||||
{"failed-folder", required_argument, 0, 'j' },
|
||||
{"move-folder", required_argument, 0, 'g' },
|
||||
{"after", required_argument, 0, 'A' },
|
||||
{"before", required_argument, 0, 'B' },
|
||||
{"only-download",no_argument, 0, 'o' },
|
||||
{"read-from-export",no_argument, 0, 'y' },
|
||||
{"dry-run", no_argument, 0, 'D' },
|
||||
{"quiet", required_argument, 0, 'q' },
|
||||
{"recursive", required_argument, 0, 'R' },
|
||||
{"remove-after-import", required_argument, 0, 'r' },
|
||||
{"help", no_argument, 0, 'h' },
|
||||
{0,0,0,0}
|
||||
};
|
||||
|
||||
int option_index = 0;
|
||||
|
||||
int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?", long_options, &option_index);
|
||||
c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:Rrqh?", long_options, &option_index);
|
||||
#else
|
||||
int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?");
|
||||
c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:Rrqh?");
|
||||
#endif
|
||||
|
||||
if(c == -1) break;
|
||||
@ -176,7 +440,7 @@ int main(int argc, char **argv){
|
||||
break;
|
||||
|
||||
case 'e' :
|
||||
snprintf(data.import->filename, SMALLBUFSIZE-1, "%s", optarg);
|
||||
emlfile = optarg;
|
||||
break;
|
||||
|
||||
case 'd' :
|
||||
@ -193,42 +457,36 @@ int main(int argc, char **argv){
|
||||
break;
|
||||
|
||||
case 'M' :
|
||||
data.import->mboxdir = optarg;
|
||||
mboxdir = optarg;
|
||||
break;
|
||||
|
||||
case 'i' :
|
||||
imapserver = optarg;
|
||||
data.import->server = optarg;
|
||||
break;
|
||||
|
||||
case 'K' :
|
||||
pop3server = optarg;
|
||||
data.import->server = optarg;
|
||||
if(data.import->port == 143) data.import->port = 110;
|
||||
if(port == 143) port = 110;
|
||||
break;
|
||||
|
||||
case 'u' :
|
||||
data.import->username = optarg;
|
||||
username = optarg;
|
||||
break;
|
||||
|
||||
case 'p' :
|
||||
data.import->password = optarg;
|
||||
password = optarg;
|
||||
break;
|
||||
|
||||
case 'P' :
|
||||
data.import->port = atoi(optarg);
|
||||
port = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'x' :
|
||||
data.import->skiplist = optarg;
|
||||
skiplist = optarg;
|
||||
break;
|
||||
|
||||
case 'F' :
|
||||
data.import->folder = optarg;
|
||||
break;
|
||||
|
||||
case 'f' :
|
||||
data.import->folder_imap = optarg;
|
||||
folder = optarg;
|
||||
break;
|
||||
|
||||
case 'R' :
|
||||
@ -236,78 +494,11 @@ int main(int argc, char **argv){
|
||||
break;
|
||||
|
||||
case 'r' :
|
||||
data.import->remove_after_import = 1;
|
||||
break;
|
||||
|
||||
case 'g' :
|
||||
data.import->move_folder = optarg;
|
||||
break;
|
||||
|
||||
case 'j' :
|
||||
data.import->failed_folder = optarg;
|
||||
break;
|
||||
|
||||
case 'o' :
|
||||
data.import->download_only = 1;
|
||||
data.import->dryrun = 1;
|
||||
break;
|
||||
|
||||
case 'b' :
|
||||
data.import->batch_processing_limit = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 't' :
|
||||
data.import->timeout = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 's' :
|
||||
if(atoi(optarg) < 1){
|
||||
printf("invalid start position: %s\n", optarg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
data.import->start_position = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'a' :
|
||||
snprintf(puf, sizeof(puf)-1, "%s ", optarg);
|
||||
data.import->extra_recipient = puf;
|
||||
break;
|
||||
|
||||
case 'T' :
|
||||
if(atoi(optarg) < 1){
|
||||
printf("invalid import table id: %s\n", optarg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
data.import->table_id = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'Z' :
|
||||
if(atoi(optarg) < 1){
|
||||
printf("invalid delay value: %s\n", optarg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
data.import->delay = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'y' :
|
||||
read_from_pilerexport = 1;
|
||||
break;
|
||||
|
||||
case 'D' :
|
||||
data.import->dryrun = 1;
|
||||
remove_after_successful_import = 1;
|
||||
break;
|
||||
|
||||
case 'q' :
|
||||
data.quiet = 1;
|
||||
break;
|
||||
|
||||
case 'A' : data.import->after = atol(optarg);
|
||||
break;
|
||||
|
||||
case 'B' : data.import->before = atol(optarg);
|
||||
quiet = 1;
|
||||
break;
|
||||
|
||||
case 'h' :
|
||||
@ -316,105 +507,85 @@ int main(int argc, char **argv){
|
||||
break;
|
||||
|
||||
|
||||
default :
|
||||
default :
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server && !read_from_pilerexport) usage();
|
||||
|
||||
if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){
|
||||
printf("cannot write failed directory '%s'\n", data.import->failed_folder);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
|
||||
if(!mbox[0] && !mboxdir && !emlfile && !directory && !imapserver && !pop3server) usage();
|
||||
|
||||
cfg = read_config(configfile);
|
||||
|
||||
memset(cfg.security_header, 0, MAXVAL);
|
||||
|
||||
if((data.recursive_folder_names == 1 || data.import->folder) && cfg.enable_folders == 0){
|
||||
printf("please set enable_folders=1 in piler.conf to use the folder options\n");
|
||||
return ERR;
|
||||
}
|
||||
|
||||
/* make sure we don't discard messages without a valid Message-Id when importing manually */
|
||||
cfg.archive_emails_not_having_message_id = 1;
|
||||
|
||||
/* The mmap_dedup_test feature is expected to work with the piler daemon only */
|
||||
cfg.mmap_dedup_test = 0;
|
||||
|
||||
if(read_key(&cfg)){
|
||||
printf("%s\n", ERR_READING_KEY);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
/* enable using the extra email address */
|
||||
if(data.import->extra_recipient) cfg.process_rcpt_to_addresses = 1;
|
||||
mysql_init(&(sdata.mysql));
|
||||
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
|
||||
if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
|
||||
printf("error: cant connect to mysql server\n");
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(open_database(&sdata, &cfg) == ERR) return 0;
|
||||
mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
|
||||
mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
|
||||
|
||||
if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR) return 0;
|
||||
if(create_prepared_statements(&sdata, &data) == ERR){
|
||||
rc = ERR;
|
||||
goto ENDE;
|
||||
}
|
||||
|
||||
setlocale(LC_CTYPE, cfg.locale);
|
||||
|
||||
(void) openlog("pilerimport", LOG_PID, LOG_MAIL);
|
||||
|
||||
#ifdef HAVE_MEMCACHED
|
||||
memcached_init(&(data.memc), cfg.memcached_servers, 11211);
|
||||
#endif
|
||||
if(folder){
|
||||
data.folder = get_folder_id(&sdata, &data, folder, 0);
|
||||
|
||||
if(data.import->folder){
|
||||
data.folder = get_folder_id(&sdata, data.import->folder, 0);
|
||||
|
||||
if(data.folder == ERR_FOLDER){
|
||||
data.folder = add_new_folder(&sdata, data.import->folder, 0);
|
||||
if(data.folder == 0){
|
||||
data.folder = add_new_folder(&sdata, &data, folder, 0);
|
||||
}
|
||||
|
||||
if(data.folder == ERR_FOLDER){
|
||||
printf("ERROR: cannot get/add folder '%s'\n", data.import->folder);
|
||||
close_database(&sdata);
|
||||
if(data.folder == 0){
|
||||
printf("error: cannot get/add folder '%s'\n", folder);
|
||||
mysql_close(&(sdata.mysql));
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
|
||||
load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
|
||||
load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
|
||||
load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
|
||||
load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
|
||||
|
||||
load_mydomains(&sdata, &data, &cfg);
|
||||
|
||||
if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg);
|
||||
if(emlfile) rc = import_message(emlfile, &sdata, &data, &cfg);
|
||||
|
||||
if(mbox[0]){
|
||||
for(i=0; i<n_mbox; i++){
|
||||
import_from_mailbox(mbox[i], &sdata, &data, &counters, &cfg);
|
||||
rc = import_from_mailbox(mbox[i], &sdata, &data, &cfg);
|
||||
}
|
||||
}
|
||||
if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg);
|
||||
if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg);
|
||||
if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg);
|
||||
if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg);
|
||||
if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg);
|
||||
if(mboxdir) rc = import_mbox_from_dir(mboxdir, &sdata, &data, &tot_msgs, &cfg);
|
||||
if(directory) rc = import_from_maildir(directory, &sdata, &data, &tot_msgs, &cfg);
|
||||
if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, port, &sdata, &data, skiplist, &cfg);
|
||||
if(pop3server && username && password) rc = import_from_pop3_server(pop3server, username, password, port, &sdata, &data, &cfg);
|
||||
|
||||
clearrules(data.archiving_rules);
|
||||
clearrules(data.retention_rules);
|
||||
clearrules(data.folder_rules);
|
||||
|
||||
clearhash(data.mydomains);
|
||||
free_rule(data.archiving_rules);
|
||||
free_rule(data.retention_rules);
|
||||
|
||||
update_counters(&sdata, &data, &counters, &cfg);
|
||||
ENDE:
|
||||
close_prepared_statements(&data);
|
||||
|
||||
syslog(LOG_PRIORITY, "server=%s, user=%s, directory=%s, imported=%lld, duplicated=%lld, discarded=%lld", data.import->server, data.import->username, directory, counters.c_rcvd, counters.c_duplicate, counters.c_ignore);
|
||||
mysql_close(&(sdata.mysql));
|
||||
|
||||
close_database(&sdata);
|
||||
if(quiet == 0) printf("\n");
|
||||
|
||||
if(cfg.rtindex) close_sphx(&sdata);
|
||||
|
||||
if(data.quiet == 0) printf("\n");
|
||||
|
||||
return 0;
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
|
||||
367
src/pilerpurge.c
Normal file
367
src/pilerpurge.c
Normal file
@ -0,0 +1,367 @@
|
||||
/*
|
||||
* pilerpurge.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
int dryrun = 0;
|
||||
|
||||
|
||||
#define SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE "SELECT `value` FROM `" SQL_OPTION_TABLE "` WHERE `key`='enable_purge'"
|
||||
#define SQL_STMT_DELETE_FROM_META_TABLE "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `id` IN ("
|
||||
#define SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `piler_id` IN ('"
|
||||
#define SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS "SELECT `piler_id`, `attachment_id`, `i` FROM `" SQL_ATTACHMENTS_VIEW "` WHERE `refcount`=0 AND `piler_id` IN ('"
|
||||
#define SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE "DELETE FROM `" SQL_ATTACHMENT_TABLE "` WHERE `id` IN ("
|
||||
|
||||
|
||||
int is_purge_allowed(struct session_data *sdata, struct __config *cfg){
|
||||
int rc=0;
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
|
||||
if(mysql_real_query(&(sdata->mysql), SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE, strlen(SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
row = mysql_fetch_row(res);
|
||||
if(row[0]){
|
||||
rc = atoi(row[0]);
|
||||
}
|
||||
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int remove_message_frame_files(char *s, char *update_meta_sql, struct session_data *sdata, struct __config *cfg){
|
||||
char *p, puf[SMALLBUFSIZE], filename[SMALLBUFSIZE];
|
||||
int n=0;
|
||||
struct stat st;
|
||||
|
||||
p = s;
|
||||
do {
|
||||
p = split(p, ' ', puf, sizeof(puf)-1);
|
||||
|
||||
if(strlen(puf) == RND_STR_LEN){
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, puf[8], puf[9], puf[10], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
|
||||
|
||||
if(stat(filename, &st)){
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, puf[RND_STR_LEN-6], puf[RND_STR_LEN-5], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
|
||||
}
|
||||
|
||||
if(dryrun == 1){
|
||||
n++;
|
||||
printf("removing messagefile: %s\n", filename);
|
||||
}
|
||||
else {
|
||||
if(unlink(filename) == 0) n++;
|
||||
}
|
||||
|
||||
}
|
||||
} while(p);
|
||||
|
||||
|
||||
update_meta_sql[strlen(update_meta_sql)-1] = ')';
|
||||
|
||||
if(dryrun == 1){
|
||||
printf("running sql query: *%s*\n\n", update_meta_sql);
|
||||
} else {
|
||||
mysql_real_query(&(sdata->mysql), update_meta_sql, strlen(update_meta_sql));
|
||||
}
|
||||
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
|
||||
int remove_attachments(char *in, struct session_data *sdata, struct __config *cfg){
|
||||
char filename[SMALLBUFSIZE];
|
||||
char *a, buf[BIGBUFSIZE-300], update_meta_sql[BIGBUFSIZE], delete_attachment_stmt[BIGBUFSIZE];
|
||||
int n=0, len;
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
struct stat st;
|
||||
|
||||
|
||||
if(strlen(in) < 10) return 0;
|
||||
|
||||
len = strlen(SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS) + strlen(in) + 2;
|
||||
|
||||
a = malloc(len);
|
||||
if(!a) return 0;
|
||||
|
||||
memset(a, 0, len);
|
||||
|
||||
in[strlen(in)-2] = ')';
|
||||
in[strlen(in)-1] = '\0';
|
||||
snprintf(a, len-1, "%s%s", SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS, in);
|
||||
|
||||
if(dryrun == 1) printf("running sql query: *%s*\n\n", a);
|
||||
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(update_meta_sql, 0, sizeof(update_meta_sql));
|
||||
memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
|
||||
|
||||
snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
|
||||
snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
|
||||
|
||||
if(mysql_real_query(&(sdata->mysql), a, strlen(a)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
if(!row[0]) continue;
|
||||
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, row[0][8], row[0][9], row[0][10], row[0][RND_STR_LEN-4], row[0][RND_STR_LEN-3], row[0][RND_STR_LEN-2], row[0][RND_STR_LEN-1], row[0], atoi(row[1]));
|
||||
if(stat(filename, &st)){
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, row[0][RND_STR_LEN-6], row[0][RND_STR_LEN-5], row[0][RND_STR_LEN-4], row[0][RND_STR_LEN-3], row[0][RND_STR_LEN-2], row[0][RND_STR_LEN-1], row[0], atoi(row[1]));
|
||||
}
|
||||
|
||||
if(dryrun == 1){
|
||||
printf("removing attachment: *%s*\n", filename);
|
||||
} else {
|
||||
unlink(filename);
|
||||
}
|
||||
|
||||
|
||||
if(row[2]){
|
||||
memcpy(&delete_attachment_stmt[strlen(delete_attachment_stmt)], row[2], strlen(row[2]));
|
||||
memcpy(&delete_attachment_stmt[strlen(delete_attachment_stmt)], ",", 1);
|
||||
}
|
||||
|
||||
if(atoi(row[1]) == 1){
|
||||
|
||||
memcpy(&update_meta_sql[strlen(update_meta_sql)], row[0], strlen(row[0]));
|
||||
memcpy(&update_meta_sql[strlen(update_meta_sql)], "','", 3);
|
||||
|
||||
if(strlen(buf) >= sizeof(buf)-RND_STR_LEN-2-1){
|
||||
if(strlen(update_meta_sql) > 10){
|
||||
update_meta_sql[strlen(update_meta_sql)-2] = ')';
|
||||
update_meta_sql[strlen(update_meta_sql)-1] = '\0';
|
||||
}
|
||||
|
||||
n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
|
||||
|
||||
if(strlen(delete_attachment_stmt) > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
|
||||
delete_attachment_stmt[strlen(delete_attachment_stmt)-1] = ')';
|
||||
if(dryrun == 1){
|
||||
printf("delete sql: *%s*\n", delete_attachment_stmt);
|
||||
} else {
|
||||
mysql_real_query(&(sdata->mysql), delete_attachment_stmt, strlen(delete_attachment_stmt));
|
||||
}
|
||||
}
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(update_meta_sql, 0, sizeof(update_meta_sql));
|
||||
memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
|
||||
|
||||
snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
|
||||
snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
|
||||
}
|
||||
|
||||
memcpy(&buf[strlen(buf)], row[0], strlen(row[0]));
|
||||
memcpy(&buf[strlen(buf)], " ", 1);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
free(a);
|
||||
|
||||
|
||||
if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+10){
|
||||
update_meta_sql[strlen(update_meta_sql)-2] = ')';
|
||||
update_meta_sql[strlen(update_meta_sql)-1] = '\0';
|
||||
|
||||
n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
|
||||
}
|
||||
|
||||
if(strlen(delete_attachment_stmt) > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
|
||||
delete_attachment_stmt[strlen(delete_attachment_stmt)-1] = ')';
|
||||
if(dryrun == 1){
|
||||
printf("delete sql: *%s*\n", delete_attachment_stmt);
|
||||
} else {
|
||||
mysql_real_query(&(sdata->mysql), delete_attachment_stmt, strlen(delete_attachment_stmt));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
|
||||
int purge_messages_without_attachment(struct session_data *sdata, struct __config *cfg){
|
||||
int purged=0;
|
||||
char s[SMALLBUFSIZE], buf[BIGBUFSIZE-300], update_meta_sql[BIGBUFSIZE];
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(update_meta_sql, 0, sizeof(update_meta_sql));
|
||||
|
||||
snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments=0", SQL_METADATA_TABLE, sdata->now);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "purge sql: *%s*", s);
|
||||
|
||||
if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
|
||||
if((char *)row[0] && (char *)row[1]){
|
||||
|
||||
memcpy(&update_meta_sql[strlen(update_meta_sql)], row[0], strlen(row[0]));
|
||||
memcpy(&update_meta_sql[strlen(update_meta_sql)], ",", 1);
|
||||
|
||||
if(strlen(buf) >= sizeof(buf)-RND_STR_LEN-2-1){
|
||||
|
||||
purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(update_meta_sql, 0, sizeof(update_meta_sql));
|
||||
|
||||
snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
|
||||
}
|
||||
|
||||
memcpy(&buf[strlen(buf)], row[1], strlen(row[1]));
|
||||
memcpy(&buf[strlen(buf)], " ", 1);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE)+5){
|
||||
purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
|
||||
}
|
||||
|
||||
return purged;
|
||||
}
|
||||
|
||||
|
||||
int purge_messages_with_attachments(struct session_data *sdata, struct __config *cfg){
|
||||
int purged=0;
|
||||
char s[SMALLBUFSIZE], idlist[BIGBUFSIZE];
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
|
||||
memset(idlist, 0, sizeof(idlist));
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT `piler_id` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments > 0", SQL_METADATA_TABLE, sdata->now);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "purge sql: *%s*", s);
|
||||
|
||||
if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
if((char *)row[0]){
|
||||
memcpy(&idlist[strlen(idlist)], row[0], strlen(row[0]));
|
||||
memcpy(&idlist[strlen(idlist)], "','", 3);
|
||||
}
|
||||
|
||||
if(strlen(idlist) >= sizeof(idlist)-2*RND_STR_LEN){
|
||||
purged += remove_attachments(idlist, sdata, cfg);
|
||||
memset(idlist, 0, sizeof(idlist));
|
||||
}
|
||||
}
|
||||
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
if(strlen(idlist) > 5){
|
||||
purged += remove_attachments(idlist, sdata, cfg);
|
||||
}
|
||||
|
||||
return purged;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int i, purged=0;
|
||||
char *configfile=CONFIG_FILE;
|
||||
struct session_data sdata;
|
||||
struct __config cfg;
|
||||
|
||||
|
||||
while((i = getopt(argc, argv, "c:dh?")) > 0){
|
||||
switch(i){
|
||||
|
||||
case 'c' :
|
||||
configfile = optarg;
|
||||
break;
|
||||
|
||||
case 'd' :
|
||||
dryrun = 1;
|
||||
break;
|
||||
|
||||
case 'h' :
|
||||
case '?' :
|
||||
default :
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
(void) openlog("pilerpurge", LOG_PID, LOG_MAIL);
|
||||
|
||||
cfg = read_config(configfile);
|
||||
|
||||
|
||||
mysql_init(&(sdata.mysql));
|
||||
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
|
||||
if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
|
||||
printf("cannot connect to mysql server\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
|
||||
mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
|
||||
|
||||
setlocale(LC_CTYPE, cfg.locale);
|
||||
|
||||
init_session_data(&sdata, &cfg);
|
||||
|
||||
i = is_purge_allowed(&sdata, &cfg);
|
||||
if(i == 1){
|
||||
purged += purge_messages_without_attachment(&sdata, &cfg);
|
||||
purged += purge_messages_with_attachments(&sdata, &cfg);
|
||||
|
||||
printf("purged: %d\n", purged);
|
||||
}
|
||||
else printf("purge is not allowed by configuration, enable_purge=%d\n", i);
|
||||
|
||||
mysql_close(&(sdata.mysql));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
299
src/pop3.c
299
src/pop3.c
@ -23,194 +23,183 @@
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int connect_to_pop3_server(struct data *data){
|
||||
char buf[MAXBUFSIZE];
|
||||
int is_last_complete_pop3_packet(char *s, int len){
|
||||
|
||||
if(data->net->use_ssl == 1){
|
||||
init_ssl_to_server(data);
|
||||
if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){
|
||||
return 1;
|
||||
}
|
||||
|
||||
recvtimeoutssl(data->net, buf, sizeof(buf));
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "USER %s\r\n", data->import->username);
|
||||
|
||||
write1(data->net, buf, strlen(buf));
|
||||
recvtimeoutssl(data->net, buf, sizeof(buf));
|
||||
int connect_to_pop3_server(int sd, char *username, char *password, int port, struct __data *data, int use_ssl){
|
||||
int n;
|
||||
char buf[MAXBUFSIZE];
|
||||
X509* server_cert;
|
||||
char *str;
|
||||
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "PASS %s\r\n", data->import->password);
|
||||
if(use_ssl == 1){
|
||||
|
||||
write1(data->net, buf, strlen(buf));
|
||||
recvtimeoutssl(data->net, buf, sizeof(buf));
|
||||
SSL_library_init();
|
||||
SSL_load_error_strings();
|
||||
|
||||
data->ctx = SSL_CTX_new(SSLv3_client_method());
|
||||
CHK_NULL(data->ctx, "internal SSL error");
|
||||
|
||||
data->ssl = SSL_new(data->ctx);
|
||||
CHK_NULL(data->ssl, "internal ssl error");
|
||||
|
||||
SSL_set_fd(data->ssl, sd);
|
||||
n = SSL_connect(data->ssl);
|
||||
CHK_SSL(n, "internal ssl error");
|
||||
|
||||
server_cert = SSL_get_peer_certificate(data->ssl);
|
||||
CHK_NULL(server_cert, "server cert error");
|
||||
|
||||
str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
|
||||
CHK_NULL(str, "error in server cert");
|
||||
OPENSSL_free(str);
|
||||
|
||||
str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
|
||||
CHK_NULL(str, "error in server cert");
|
||||
OPENSSL_free(str);
|
||||
|
||||
X509_free(server_cert);
|
||||
}
|
||||
|
||||
|
||||
n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
|
||||
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "USER %s\r\n", username);
|
||||
|
||||
write1(sd, buf, use_ssl, data->ssl);
|
||||
n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
|
||||
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "PASS %s\r\n", password);
|
||||
|
||||
write1(sd, buf, use_ssl, data->ssl);
|
||||
n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
|
||||
|
||||
if(strncmp(buf, "+OK", 3) == 0) return OK;
|
||||
|
||||
printf("error: %s", buf);
|
||||
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
void get_number_of_total_messages(struct data *data){
|
||||
char buf[MAXBUFSIZE];
|
||||
|
||||
data->import->total_messages = 0;
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "STAT\r\n");
|
||||
write1(data->net, buf, strlen(buf));
|
||||
|
||||
recvtimeoutssl(data->net, buf, sizeof(buf));
|
||||
|
||||
if(strncmp(buf, "+OK ", 4) == 0){
|
||||
char *p = strchr(&buf[4], ' ');
|
||||
if(p){
|
||||
*p = '\0';
|
||||
data->import->total_messages = atoi(&buf[4]);
|
||||
}
|
||||
}
|
||||
else {
|
||||
printf("ERROR: '%s'", buf);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int pop3_download_email(struct data *data, int i){
|
||||
char *p, buf[MAXBUFSIZE], savedbuf[MAXBUFSIZE], copybuf[2*MAXBUFSIZE];
|
||||
|
||||
data->import->processed_messages++;
|
||||
|
||||
snprintf(data->import->filename, SMALLBUFSIZE-1, "pop3-tmp-%d-%d.txt", getpid(), i);
|
||||
unlink(data->import->filename);
|
||||
|
||||
int fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
|
||||
if(fd == -1){
|
||||
printf("cannot open: %s\n", data->import->filename);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
memset(savedbuf, 0, sizeof(savedbuf));
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
|
||||
write1(data->net, buf, strlen(buf));
|
||||
|
||||
int nlines = 0;
|
||||
int endofmessage = 0;
|
||||
int savedlen = 0;
|
||||
int n = 0;
|
||||
|
||||
while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){
|
||||
int remaininglen = n;
|
||||
|
||||
if(savedlen){
|
||||
memset(copybuf, 0, sizeof(copybuf));
|
||||
memcpy(copybuf, savedbuf, savedlen);
|
||||
memcpy(©buf[savedlen], buf, n);
|
||||
|
||||
remaininglen += savedlen;
|
||||
|
||||
savedlen = 0;
|
||||
memset(savedbuf, 0, sizeof(savedbuf));
|
||||
|
||||
p = ©buf[0];
|
||||
} else {
|
||||
p = &buf[0];
|
||||
}
|
||||
|
||||
int puflen=0;
|
||||
int rc=OK;
|
||||
int nullbyte=0;
|
||||
|
||||
do {
|
||||
char puf[MAXBUFSIZE];
|
||||
|
||||
puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
|
||||
remaininglen -= puflen;
|
||||
nlines++;
|
||||
|
||||
if(nlines == 1){
|
||||
if(strncmp(puf, "+OK", 3)){
|
||||
printf("error: %s", puf);
|
||||
return ERR;
|
||||
}
|
||||
} else {
|
||||
if(puf[puflen-1] == '\n'){
|
||||
if(puflen == 3 && puf[0] == '.' && puf[1] == '\r' && puf[2] == '\n'){
|
||||
endofmessage = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
int dotstuff = 0;
|
||||
if(puf[0] == '.' && puf[1] != '\r' && puf[1] != '\n') dotstuff = 1;
|
||||
|
||||
if(write(fd, &puf[dotstuff], puflen-dotstuff) == -1) printf("ERROR: writing to fd\n");
|
||||
|
||||
} else if(puflen > 0) {
|
||||
savedlen = puflen;
|
||||
snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
|
||||
}
|
||||
}
|
||||
|
||||
p += puflen;
|
||||
|
||||
} while(puflen > 0);
|
||||
|
||||
if(endofmessage){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
close(fd);
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
void pop3_delete_message(struct data *data, int i){
|
||||
char buf[SMALLBUFSIZE];
|
||||
int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg){
|
||||
int i, rc=ERR, n, messages=0, processed_messages=0, pos, readlen, fd, lastpos, nreads;
|
||||
char *p, buf[MAXBUFSIZE], filename[SMALLBUFSIZE];
|
||||
char aggrbuf[3*MAXBUFSIZE];
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "DELE %d\r\n", i);
|
||||
write1(data->net, buf, strlen(buf));
|
||||
recvtimeoutssl(data->net, buf, sizeof(buf));
|
||||
}
|
||||
snprintf(buf, sizeof(buf)-1, "STAT\r\n");
|
||||
n = write1(sd, buf, use_ssl, data->ssl);
|
||||
|
||||
n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
|
||||
|
||||
if(strncmp(buf, "+OK ", 4) == 0){
|
||||
p = strchr(&buf[4], ' ');
|
||||
if(p){
|
||||
*p = '\0';
|
||||
messages = atoi(&buf[4]);
|
||||
}
|
||||
}
|
||||
else return ERR;
|
||||
|
||||
|
||||
void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
|
||||
char buf[MAXBUFSIZE];
|
||||
printf("found %d messages\n", messages);
|
||||
|
||||
data->import->processed_messages = 0;
|
||||
if(messages <= 0) return rc;
|
||||
|
||||
get_number_of_total_messages(data);
|
||||
for(i=1; i<=messages; i++){
|
||||
processed_messages++;
|
||||
printf("processed: %7d\r", processed_messages); fflush(stdout);
|
||||
|
||||
if(data->quiet == 0) printf("found %d messages\n", data->import->total_messages);
|
||||
|
||||
if(data->import->total_messages <= 0) return;
|
||||
snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
|
||||
|
||||
for(int i=data->import->start_position; i<=data->import->total_messages; i++){
|
||||
if(pop3_download_email(data, i) == OK){
|
||||
if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
|
||||
snprintf(filename, sizeof(filename)-1, "pop3-tmp-%d.txt", i);
|
||||
unlink(filename);
|
||||
|
||||
if(data->import->dryrun == 0){
|
||||
int rc = import_message(sdata, data, counters, cfg);
|
||||
fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
|
||||
if(fd == -1){
|
||||
printf("cannot open: %s\n", filename);
|
||||
return rc;
|
||||
}
|
||||
|
||||
if(data->import->remove_after_import == 1 && rc == OK){
|
||||
pop3_delete_message(data, i);
|
||||
n = write1(sd, buf, use_ssl, data->ssl);
|
||||
|
||||
readlen = 0;
|
||||
pos = 0;
|
||||
nreads = 0;
|
||||
|
||||
memset(aggrbuf, 0, sizeof(aggrbuf));
|
||||
lastpos = 0;
|
||||
|
||||
|
||||
while((n = recvtimeoutssl(sd, buf, sizeof(buf), 15, use_ssl, data->ssl)) > 0){
|
||||
nreads++;
|
||||
readlen += n;
|
||||
|
||||
if(nreads == 1){
|
||||
|
||||
if(strncmp(buf, "+OK", 3) == 0){
|
||||
p = strchr(&buf[3], '\n');
|
||||
if(p){
|
||||
*p = '\0';
|
||||
pos = strlen(buf)+1;
|
||||
*p = '\n';
|
||||
}
|
||||
}
|
||||
else { printf("error: %s", buf); return ERR; }
|
||||
|
||||
}
|
||||
|
||||
if(lastpos + 1 + n < sizeof(aggrbuf)){
|
||||
|
||||
if(nreads == 1){
|
||||
memcpy(aggrbuf+lastpos, buf+pos, n-pos);
|
||||
lastpos += n-pos;
|
||||
}
|
||||
else {
|
||||
memcpy(aggrbuf+lastpos, buf, n);
|
||||
lastpos += n;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
write(fd, aggrbuf, sizeof(buf));
|
||||
|
||||
if(data->import->download_only == 0) unlink(data->import->filename);
|
||||
memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
|
||||
lastpos -= sizeof(buf);
|
||||
|
||||
/* whether to quit after processing a batch of messages */
|
||||
memcpy(aggrbuf+lastpos, buf, n);
|
||||
lastpos += n;
|
||||
}
|
||||
|
||||
if(data->import->batch_processing_limit > 0 && data->import->processed_messages >= data->import->batch_processing_limit){
|
||||
break;
|
||||
}
|
||||
if(is_last_complete_pop3_packet(aggrbuf, lastpos) == 1){
|
||||
write(fd, aggrbuf, lastpos-3);
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
close(fd);
|
||||
|
||||
rc = import_message(filename, sdata, data, cfg);
|
||||
|
||||
unlink(filename);
|
||||
}
|
||||
|
||||
|
||||
snprintf(buf, sizeof(buf)-1, "QUIT\r\n");
|
||||
write1(data->net, buf, strlen(buf));
|
||||
n = write1(sd, buf, use_ssl, data->ssl);
|
||||
|
||||
if(data->quiet == 0) printf("\n");
|
||||
printf("\n");
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user