mirror of
https://bitbucket.org/jsuto/piler.git
synced 2024-12-24 19:30:12 +01:00
Added support for manticore
Signed-off-by: Janos SUTO <sj@acts.hu>
This commit is contained in:
Janos SUTO
parent
507ac86885
commit
c0859cdd65
@ -32,7 +32,9 @@ all:
|
||||
install:
|
||||
$(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist
|
||||
sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
|
||||
sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist
|
||||
$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
|
||||
$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist
|
||||
$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php
|
||||
$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron
|
||||
sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
|
||||
|
||||
@ -2,8 +2,8 @@
|
||||
5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
|
||||
30 2 * * * LIBEXECDIR/piler/indexer.main.sh
|
||||
40 3 * * * LIBEXECDIR/piler/purge.sh
|
||||
*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate
|
||||
*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
|
||||
*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate
|
||||
*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate
|
||||
*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
|
||||
*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
|
||||
|
||||
|
||||
199
etc/manticore.conf.in
Executable file
199
etc/manticore.conf.in
Executable file
@ -0,0 +1,199 @@
|
||||
#!/usr/bin/php
|
||||
<?php
|
||||
|
||||
define('LOCALSTATEDIR', '/var');
|
||||
define('NGRAM_CONFIG', " #ngram_len = 1\n #ngram_chars = U+3000..U+2FA1F\n");
|
||||
|
||||
# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
|
||||
# The following settings contains English and some Latin extras
|
||||
define('CHARSET_TABLE', "0..9, english, _, \
|
||||
U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
|
||||
U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
|
||||
U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
|
||||
U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
|
||||
|
||||
define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
|
||||
|
||||
?>
|
||||
|
||||
#
|
||||
# minimal sphinx configuration suited to piler
|
||||
#
|
||||
|
||||
source base
|
||||
{
|
||||
type = mysql
|
||||
sql_host = MYSQL_HOSTNAME
|
||||
sql_db = MYSQL_DATABASE
|
||||
sql_user = MYSQL_USERNAME
|
||||
sql_pass = MYSQL_PASSWORD
|
||||
|
||||
sql_attr_uint = size
|
||||
sql_attr_uint = sent
|
||||
sql_attr_uint = attachments
|
||||
}
|
||||
|
||||
source delta : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query_pre = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
|
||||
sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
|
||||
|
||||
sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
|
||||
}
|
||||
|
||||
source main1 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source main2 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source main3 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source main4 : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source dailydelta : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
|
||||
}
|
||||
|
||||
source tag : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
|
||||
|
||||
sql_attr_uint = iid
|
||||
sql_attr_uint = uid
|
||||
|
||||
}
|
||||
|
||||
source note : base
|
||||
{
|
||||
sql_query_pre = SET NAMES utf8mb4
|
||||
sql_query = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
|
||||
|
||||
sql_attr_uint = iid
|
||||
sql_attr_uint = uid
|
||||
|
||||
}
|
||||
|
||||
|
||||
index main1
|
||||
{
|
||||
source = main1
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/main1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index main2
|
||||
{
|
||||
source = main2
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/main2
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index main3
|
||||
{
|
||||
source = main3
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/main3
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index main4
|
||||
{
|
||||
source = main4
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/main4
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index dailydelta1
|
||||
{
|
||||
source = dailydelta
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/dailydelta1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
index delta1
|
||||
{
|
||||
source = delta
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/delta1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
|
||||
index tag1
|
||||
{
|
||||
source = tag
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/tag1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
|
||||
index note1
|
||||
{
|
||||
source = note
|
||||
path = <?php print LOCALSTATEDIR; ?>/piler/manticore/note1
|
||||
min_prefix_len = 5
|
||||
min_word_len = 1
|
||||
charset_table = <?php print CHARSET_TABLE; ?>
|
||||
<?php print NGRAM_CONFIG; ?>
|
||||
}
|
||||
|
||||
|
||||
indexer
|
||||
{
|
||||
mem_limit = 256M
|
||||
}
|
||||
|
||||
|
||||
searchd
|
||||
{
|
||||
listen = 127.0.0.1:9312
|
||||
listen = 127.0.0.1:9306:mysql41
|
||||
log = /var/piler/manticore/manticore.log
|
||||
binlog_path = /var/piler/manticore/
|
||||
query_log = /var/piler/manticore/query.log
|
||||
network_timeout = 5
|
||||
pid_file = /var/run/piler/searchd.pid
|
||||
seamless_rotate = 1
|
||||
preopen_indexes = 1
|
||||
unlink_old = 1
|
||||
thread_stack = 512k
|
||||
}
|
||||
@ -16,7 +16,13 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
|
||||
|
||||
NAME=searchd
|
||||
PID_FILE=/var/run/piler/searchd.pid
|
||||
PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
|
||||
PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE})
|
||||
CONFIG_FILE=sphinx.conf
|
||||
|
||||
if [ -f SYSCONFDIR/piler/MANTICORE ]; then
|
||||
CONFIG_FILE=manticore.conf
|
||||
fi
|
||||
|
||||
|
||||
start() {
|
||||
echo "starting searchd . . ."
|
||||
@ -26,8 +32,8 @@ start() {
|
||||
chown piler:piler /var/run/piler
|
||||
fi
|
||||
|
||||
if [ `id -u` -eq 0 ]; then
|
||||
su piler -c "searchd --config SYSCONFDIR/piler/sphinx.conf"
|
||||
if [ $(id -u) -eq 0 ]; then
|
||||
su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}"
|
||||
else
|
||||
searchd
|
||||
fi
|
||||
@ -35,7 +41,7 @@ start() {
|
||||
|
||||
stop() {
|
||||
echo "stopping searchd"
|
||||
kill ${PID_NUMBER}
|
||||
kill "$PID_NUMBER"
|
||||
}
|
||||
|
||||
check_status(){
|
||||
|
||||
@ -6,10 +6,17 @@ DELTATMPFILE=/var/run/piler/delta.indexer.tmp
|
||||
INDEXER="indexer --config SYSCONFDIR/piler/sphinx.conf"
|
||||
PRIORITY=mail.info
|
||||
TOUCHFILE=/var/piler/stat/indexer
|
||||
CONFIG_FILE=sphinx.conf
|
||||
|
||||
if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi
|
||||
if [[ -f SYSCONFDIR/piler/MANTICORE ]]; then
|
||||
CONFIG_FILE=manticore.conf
|
||||
fi
|
||||
|
||||
if [ -f $DELTATMPFILE ]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at "`cat $DELTATMPFILE` | logger -p $PRIORITY ; exit 1; fi
|
||||
INDEXER="indexer --config SYSCONFDIR/piler/${CONFIG_FILE}"
|
||||
|
||||
if [[ -f "$MAINTMPFILE" ]]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at $(cat "$MAINTMPFILE")" | logger -p "$PRIORITY" ; exit 1; fi
|
||||
|
||||
if [[ -f "$DELTATMPFILE" ]]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at $(cat "$DELTATMPFILE")" | logger -p "$PRIORITY" ; exit 1; fi
|
||||
|
||||
date > $DELTATMPFILE
|
||||
|
||||
@ -34,4 +41,3 @@ echo "INDEXER INFO: merging delta to dailydelta started" | logger -p $PRIORITY
|
||||
$INDEXER --quiet --merge dailydelta1 delta1 --merge-dst-range deleted 0 0 --rotate
|
||||
|
||||
echo "INDEXER INFO: merging delta to dailydelta finished" | logger -p $PRIORITY
|
||||
|
||||
|
||||
@ -5,11 +5,17 @@ set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
MAINTMPFILE="/var/run/piler/main.indexer.tmp"
|
||||
SPHINX_CONFIG="SYSCONFDIR/piler/sphinx.conf"
|
||||
CONFIG_FILE="SYSCONFDIR/piler/sphinx.conf"
|
||||
INDEXDIR=sphinx
|
||||
PRIORITY="mail.info"
|
||||
TOUCHFILE="/var/piler/stat/indexer"
|
||||
MAIN_INDEX="main1"
|
||||
|
||||
if [[ -f SYSCONFDIR/piler/MANTICORE ]]; then
|
||||
CONFIG_FILE=SYSCONFDIR/piler/manticore.conf
|
||||
INDEXDIR=manticore
|
||||
fi
|
||||
|
||||
export PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
|
||||
|
||||
finish() {
|
||||
@ -30,7 +36,7 @@ trap finish EXIT
|
||||
|
||||
echo "INDEXER INFO: merging to main started" | logger -p "$PRIORITY"
|
||||
|
||||
indexer --config "$SPHINX_CONFIG" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate
|
||||
indexer --config "$CONFIG_FILE" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate
|
||||
|
||||
echo "INDEXER INFO: merging to main finished" | logger -p "$PRIORITY"
|
||||
|
||||
@ -38,12 +44,12 @@ sleep 5
|
||||
|
||||
echo "INDEXER INFO: resetting daily delta started" | logger -p "$PRIORITY"
|
||||
|
||||
indexer --config "$SPHINX_CONFIG" --quiet dailydelta1 --rotate
|
||||
indexer --config "$CONFIG_FILE" --quiet dailydelta1 --rotate
|
||||
|
||||
echo "INDEXER INFO: resetting daily delta finished" | logger -p "$PRIORITY"
|
||||
|
||||
sum=0
|
||||
while read -r a b; do
|
||||
sum=$(( sum + b ))
|
||||
done < <( find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" )
|
||||
done < <( find /var/piler/${INDEXDIR}/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" )
|
||||
printf "%d" $sum > /var/piler/stat/main_index_size
|
||||
|
||||
@ -36,7 +36,7 @@ load_default_values() {
|
||||
MYSQL_ROOT_PASSWORD=""
|
||||
MYSQL_SOCKET=""
|
||||
|
||||
SPHINXCFG="${PILER_CONFIG_DIR}/sphinx.conf"
|
||||
SEARCHCFG="${PILER_CONFIG_DIR}/manticore.conf"
|
||||
|
||||
WWWGROUP="apache"
|
||||
DOCROOT="/var/piler/www"
|
||||
@ -196,9 +196,9 @@ askYN() {
|
||||
preinstall_check() {
|
||||
check_user root
|
||||
|
||||
if [[ "$INDEXER" == "" ]]; then "ERROR: cannot find sphinx indexer"; echo ""; exit ; fi
|
||||
if [[ "$INDEXER" == "" ]]; then "ERROR: cannot find indexer binary"; echo ""; exit ; fi
|
||||
|
||||
if [[ "$SEARCHD" == "" ]]; then "ERROR: cannot find sphinx searchd"; echo ""; exit 0; fi
|
||||
if [[ "$SEARCHD" == "" ]]; then "ERROR: cannot find searchd binary"; echo ""; exit 0; fi
|
||||
|
||||
if [[ -f "$KEYFILE" ]]; then echo "ERROR: found existing keyfile (${KEYFILE}), aborting install"; echo ""; exit 0; fi
|
||||
}
|
||||
@ -281,8 +281,8 @@ make_cron_entries() {
|
||||
echo "30 2 * * * ${LIBEXECDIR}/piler/indexer.main.sh";
|
||||
echo "40 3 * * * ${LIBEXECDIR}/piler/purge.sh";
|
||||
echo "3 * * * * ${LIBEXECDIR}/piler/watch_sphinx_main_index.sh";
|
||||
echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${PILER_CONFIG_DIR}/sphinx.conf";
|
||||
echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${PILER_CONFIG_DIR}/sphinx.conf";
|
||||
echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${PILER_CONFIG_DIR}/manticore.conf";
|
||||
echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${PILER_CONFIG_DIR}/manticore.conf";
|
||||
echo "30 6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null";
|
||||
echo "*/5 * * * * /usr/bin/find ${LOCALSTATEDIR}/piler/error -type f|wc -l > ${LOCALSTATEDIR}/piler/stat/error";
|
||||
echo "*/5 * * * * /usr/bin/find ${DOCROOT}/tmp -type f -name i.\* -exec rm -f {} \;";
|
||||
@ -316,8 +316,8 @@ show_summary() {
|
||||
echo "mysql password: *******"
|
||||
echo
|
||||
|
||||
echo "sphinx indexer: ${INDEXER}"
|
||||
echo "sphinx config file: ${SPHINXCFG}"
|
||||
echo "indexer: ${INDEXER}"
|
||||
echo "config file: ${SEARCHCFG}"
|
||||
echo
|
||||
|
||||
echo "vhost docroot: ${DOCROOT}"
|
||||
@ -358,14 +358,12 @@ execute_post_install_tasks() {
|
||||
echo "Done."
|
||||
|
||||
echo -n "Writing sphinx configuration... ";
|
||||
sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" -e "s%MYSQL_USERNAME%${MYSQL_USERNAME}%" -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" "${SYSCONFDIR}/piler/sphinx.conf.dist" > "$SPHINXCFG"
|
||||
chmod +x "$SPHINXCFG"
|
||||
sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" -e "s%MYSQL_USERNAME%${MYSQL_USERNAME}%" -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" "${SYSCONFDIR}/piler/manticore.conf.dist" > "$SEARCHCFG"
|
||||
chmod +x "$SEARCHCFG"
|
||||
echo "Done."
|
||||
|
||||
if indexer --version | grep ^Sphinx\ 3; then sed -i 's/220/311/' "$SPHINXCFG"; fi
|
||||
|
||||
echo -n "Initializing sphinx indices... ";
|
||||
su "$PILERUSER" -c "indexer --all --config ${SPHINXCFG}"
|
||||
echo -n "Initializing manticore indices... ";
|
||||
su "$PILERUSER" -c "indexer --all --config ${SEARCHCFG}"
|
||||
echo "Done."
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user