From c0859cdd65cb8ee11ef436f260c8ab573e8f0e62 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 27 Aug 2022 11:03:29 +0200 Subject: [PATCH] Added support for manticore Signed-off-by: Janos SUTO --- etc/Makefile.in | 2 + etc/cron.jobs.in | 4 +- etc/manticore.conf.in | 199 +++++++++++++++++++++++++++++++++++++++ init.d/rc.searchd.in | 14 ++- util/indexer.delta.sh.in | 12 ++- util/indexer.main.sh.in | 14 ++- util/postinstall.sh.in | 24 +++-- 7 files changed, 243 insertions(+), 26 deletions(-) create mode 100755 etc/manticore.conf.in diff --git a/etc/Makefile.in b/etc/Makefile.in index e05ab252..48b535fe 100644 --- a/etc/Makefile.in +++ b/etc/Makefile.in @@ -32,7 +32,9 @@ all: install: $(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist + sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist $(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist + $(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist $(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php $(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron diff --git a/etc/cron.jobs.in b/etc/cron.jobs.in index 5741c047..cfd95892 100644 --- a/etc/cron.jobs.in +++ b/etc/cron.jobs.in @@ -2,8 +2,8 @@ 5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh 30 2 * * * LIBEXECDIR/piler/indexer.main.sh 40 3 * * * LIBEXECDIR/piler/purge.sh -*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate -*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate +*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate +*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \; */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error diff --git a/etc/manticore.conf.in b/etc/manticore.conf.in new file mode 100755 index 00000000..ec24f12e --- /dev/null +++ b/etc/manticore.conf.in @@ -0,0 +1,199 @@ +#!/usr/bin/php +U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \ + U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \ + U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \ + U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n"); + +define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`'); + +?> + +# +# minimal sphinx configuration suited to piler +# + +source base +{ + type = mysql + sql_host = MYSQL_HOSTNAME + sql_db = MYSQL_DATABASE + sql_user = MYSQL_USERNAME + sql_pass = MYSQL_PASSWORD + + sql_attr_uint = size + sql_attr_uint = sent + sql_attr_uint = attachments +} + +source delta : base +{ + sql_query_pre = SET NAMES utf8mb4 + sql_query_pre = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index + sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1) + sql_query = SELECT FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1) + + sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1 +} + +source main1 : base +{ + sql_query_pre = SET NAMES utf8mb4 + sql_query = SELECT FROM sph_index WHERE id=-1 +} + +source main2 : base +{ + sql_query_pre = SET NAMES utf8mb4 + sql_query = SELECT FROM sph_index WHERE id=-1 +} + +source main3 : base +{ + sql_query_pre = SET NAMES utf8mb4 + sql_query = SELECT FROM sph_index WHERE id=-1 +} + +source main4 : base +{ + sql_query_pre = SET NAMES utf8mb4 + sql_query = SELECT FROM sph_index WHERE id=-1 +} + +source dailydelta : base +{ + sql_query_pre = SET NAMES utf8mb4 + sql_query = SELECT FROM sph_index WHERE id=-1 +} + +source tag : base +{ + sql_query_pre = SET NAMES utf8mb4 + sql_query = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag` + + sql_attr_uint = iid + sql_attr_uint = uid + +} + +source note : base +{ + sql_query_pre = SET NAMES utf8mb4 + sql_query = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note` + + sql_attr_uint = iid + sql_attr_uint = uid + +} + + +index main1 +{ + source = main1 + path = /piler/manticore/main1 + min_prefix_len = 5 + min_word_len = 1 + charset_table = + +} + +index main2 +{ + source = main2 + path = /piler/manticore/main2 + min_prefix_len = 5 + min_word_len = 1 + charset_table = + +} + +index main3 +{ + source = main3 + path = /piler/manticore/main3 + min_prefix_len = 5 + min_word_len = 1 + charset_table = + +} + +index main4 +{ + source = main4 + path = /piler/manticore/main4 + min_prefix_len = 5 + min_word_len = 1 + charset_table = + +} + +index dailydelta1 +{ + source = dailydelta + path = /piler/manticore/dailydelta1 + min_prefix_len = 5 + min_word_len = 1 + charset_table = + +} + +index delta1 +{ + source = delta + path = /piler/manticore/delta1 + min_prefix_len = 5 + min_word_len = 1 + charset_table = + +} + + +index tag1 +{ + source = tag + path = /piler/manticore/tag1 + min_prefix_len = 5 + min_word_len = 1 + charset_table = + +} + + +index note1 +{ + source = note + path = /piler/manticore/note1 + min_prefix_len = 5 + min_word_len = 1 + charset_table = + +} + + +indexer +{ + mem_limit = 256M +} + + +searchd +{ + listen = 127.0.0.1:9312 + listen = 127.0.0.1:9306:mysql41 + log = /var/piler/manticore/manticore.log + binlog_path = /var/piler/manticore/ + query_log = /var/piler/manticore/query.log + network_timeout = 5 + pid_file = /var/run/piler/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + thread_stack = 512k +} diff --git a/init.d/rc.searchd.in b/init.d/rc.searchd.in index 5acb09d3..e2cf95cf 100644 --- a/init.d/rc.searchd.in +++ b/init.d/rc.searchd.in @@ -16,7 +16,13 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin NAME=searchd PID_FILE=/var/run/piler/searchd.pid -PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}` +PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE}) +CONFIG_FILE=sphinx.conf + +if [ -f SYSCONFDIR/piler/MANTICORE ]; then + CONFIG_FILE=manticore.conf +fi + start() { echo "starting searchd . . ." @@ -26,8 +32,8 @@ start() { chown piler:piler /var/run/piler fi - if [ `id -u` -eq 0 ]; then - su piler -c "searchd --config SYSCONFDIR/piler/sphinx.conf" + if [ $(id -u) -eq 0 ]; then + su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}" else searchd fi @@ -35,7 +41,7 @@ start() { stop() { echo "stopping searchd" - kill ${PID_NUMBER} + kill "$PID_NUMBER" } check_status(){ diff --git a/util/indexer.delta.sh.in b/util/indexer.delta.sh.in index 6b05fe6f..ee58e04f 100755 --- a/util/indexer.delta.sh.in +++ b/util/indexer.delta.sh.in @@ -6,10 +6,17 @@ DELTATMPFILE=/var/run/piler/delta.indexer.tmp INDEXER="indexer --config SYSCONFDIR/piler/sphinx.conf" PRIORITY=mail.info TOUCHFILE=/var/piler/stat/indexer +CONFIG_FILE=sphinx.conf -if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi +if [[ -f SYSCONFDIR/piler/MANTICORE ]]; then + CONFIG_FILE=manticore.conf +fi -if [ -f $DELTATMPFILE ]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at "`cat $DELTATMPFILE` | logger -p $PRIORITY ; exit 1; fi +INDEXER="indexer --config SYSCONFDIR/piler/${CONFIG_FILE}" + +if [[ -f "$MAINTMPFILE" ]]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at $(cat "$MAINTMPFILE")" | logger -p "$PRIORITY" ; exit 1; fi + +if [[ -f "$DELTATMPFILE" ]]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at $(cat "$DELTATMPFILE")" | logger -p "$PRIORITY" ; exit 1; fi date > $DELTATMPFILE @@ -34,4 +41,3 @@ echo "INDEXER INFO: merging delta to dailydelta started" | logger -p $PRIORITY $INDEXER --quiet --merge dailydelta1 delta1 --merge-dst-range deleted 0 0 --rotate echo "INDEXER INFO: merging delta to dailydelta finished" | logger -p $PRIORITY - diff --git a/util/indexer.main.sh.in b/util/indexer.main.sh.in index 7332ed48..0f4e9653 100755 --- a/util/indexer.main.sh.in +++ b/util/indexer.main.sh.in @@ -5,11 +5,17 @@ set -o errexit set -o pipefail MAINTMPFILE="/var/run/piler/main.indexer.tmp" -SPHINX_CONFIG="SYSCONFDIR/piler/sphinx.conf" +CONFIG_FILE="SYSCONFDIR/piler/sphinx.conf" +INDEXDIR=sphinx PRIORITY="mail.info" TOUCHFILE="/var/piler/stat/indexer" MAIN_INDEX="main1" +if [[ -f SYSCONFDIR/piler/MANTICORE ]]; then + CONFIG_FILE=SYSCONFDIR/piler/manticore.conf + INDEXDIR=manticore +fi + export PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" finish() { @@ -30,7 +36,7 @@ trap finish EXIT echo "INDEXER INFO: merging to main started" | logger -p "$PRIORITY" -indexer --config "$SPHINX_CONFIG" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate +indexer --config "$CONFIG_FILE" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate echo "INDEXER INFO: merging to main finished" | logger -p "$PRIORITY" @@ -38,12 +44,12 @@ sleep 5 echo "INDEXER INFO: resetting daily delta started" | logger -p "$PRIORITY" -indexer --config "$SPHINX_CONFIG" --quiet dailydelta1 --rotate +indexer --config "$CONFIG_FILE" --quiet dailydelta1 --rotate echo "INDEXER INFO: resetting daily delta finished" | logger -p "$PRIORITY" sum=0 while read -r a b; do sum=$(( sum + b )) -done < <( find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" ) +done < <( find /var/piler/${INDEXDIR}/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" ) printf "%d" $sum > /var/piler/stat/main_index_size diff --git a/util/postinstall.sh.in b/util/postinstall.sh.in index 67c9ff41..7a2b8f84 100755 --- a/util/postinstall.sh.in +++ b/util/postinstall.sh.in @@ -36,7 +36,7 @@ load_default_values() { MYSQL_ROOT_PASSWORD="" MYSQL_SOCKET="" - SPHINXCFG="${PILER_CONFIG_DIR}/sphinx.conf" + SEARCHCFG="${PILER_CONFIG_DIR}/manticore.conf" WWWGROUP="apache" DOCROOT="/var/piler/www" @@ -196,9 +196,9 @@ askYN() { preinstall_check() { check_user root - if [[ "$INDEXER" == "" ]]; then "ERROR: cannot find sphinx indexer"; echo ""; exit ; fi + if [[ "$INDEXER" == "" ]]; then "ERROR: cannot find indexer binary"; echo ""; exit ; fi - if [[ "$SEARCHD" == "" ]]; then "ERROR: cannot find sphinx searchd"; echo ""; exit 0; fi + if [[ "$SEARCHD" == "" ]]; then "ERROR: cannot find searchd binary"; echo ""; exit 0; fi if [[ -f "$KEYFILE" ]]; then echo "ERROR: found existing keyfile (${KEYFILE}), aborting install"; echo ""; exit 0; fi } @@ -281,8 +281,8 @@ make_cron_entries() { echo "30 2 * * * ${LIBEXECDIR}/piler/indexer.main.sh"; echo "40 3 * * * ${LIBEXECDIR}/piler/purge.sh"; echo "3 * * * * ${LIBEXECDIR}/piler/watch_sphinx_main_index.sh"; - echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${PILER_CONFIG_DIR}/sphinx.conf"; - echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${PILER_CONFIG_DIR}/sphinx.conf"; + echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${PILER_CONFIG_DIR}/manticore.conf"; + echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${PILER_CONFIG_DIR}/manticore.conf"; echo "30 6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null"; echo "*/5 * * * * /usr/bin/find ${LOCALSTATEDIR}/piler/error -type f|wc -l > ${LOCALSTATEDIR}/piler/stat/error"; echo "*/5 * * * * /usr/bin/find ${DOCROOT}/tmp -type f -name i.\* -exec rm -f {} \;"; @@ -316,8 +316,8 @@ show_summary() { echo "mysql password: *******" echo - echo "sphinx indexer: ${INDEXER}" - echo "sphinx config file: ${SPHINXCFG}" + echo "indexer: ${INDEXER}" + echo "config file: ${SEARCHCFG}" echo echo "vhost docroot: ${DOCROOT}" @@ -358,14 +358,12 @@ execute_post_install_tasks() { echo "Done." echo -n "Writing sphinx configuration... "; - sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" -e "s%MYSQL_USERNAME%${MYSQL_USERNAME}%" -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" "${SYSCONFDIR}/piler/sphinx.conf.dist" > "$SPHINXCFG" - chmod +x "$SPHINXCFG" + sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" -e "s%MYSQL_USERNAME%${MYSQL_USERNAME}%" -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" "${SYSCONFDIR}/piler/manticore.conf.dist" > "$SEARCHCFG" + chmod +x "$SEARCHCFG" echo "Done." - if indexer --version | grep ^Sphinx\ 3; then sed -i 's/220/311/' "$SPHINXCFG"; fi - - echo -n "Initializing sphinx indices... "; - su "$PILERUSER" -c "indexer --all --config ${SPHINXCFG}" + echo -n "Initializing manticore indices... "; + su "$PILERUSER" -c "indexer --all --config ${SEARCHCFG}" echo "Done."