Added support for manticore

Signed-off-by: Janos SUTO <sj@acts.hu>

etc/Makefile.in

@@ -32,7 +32,9 @@ all:
 install:
 	$(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist
 	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
+	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist
 	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
+	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron
 	sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron

etc/cron.jobs.in

@@ -2,8 +2,8 @@
 5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
 30   2 * * * LIBEXECDIR/piler/indexer.main.sh
 40 3 * * * LIBEXECDIR/piler/purge.sh
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate
 */5 * * * *  /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
 */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
 

etc/manticore.conf.in

@@ -0,0 +1,199 @@
+#!/usr/bin/php
+<?php
+
+define('LOCALSTATEDIR', '/var');
+define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
+
+# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
+# The following settings contains English and some Latin extras
+define('CHARSET_TABLE', "0..9, english, _, \
+                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
+                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
+                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
+                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
+
+define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+
+?>
+
+#
+# minimal sphinx configuration suited to piler
+#
+
+source base
+{
+   type = mysql
+   sql_host = MYSQL_HOSTNAME
+   sql_db = MYSQL_DATABASE
+   sql_user = MYSQL_USERNAME
+   sql_pass = MYSQL_PASSWORD
+
+   sql_attr_uint = size
+   sql_attr_uint = sent
+   sql_attr_uint = attachments
+}
+
+source delta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
+   sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+
+   sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
+}
+
+source main1 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main2 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main3 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main4 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source dailydelta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source tag : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+
+}
+
+source note : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+
+}
+
+
+index main1
+{
+        source                  = main1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main2
+{
+        source                  = main2
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main2
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main3
+{
+        source                  = main3
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main3
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main4
+{
+        source                  = main4
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main4
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index dailydelta1
+{
+        source                  = dailydelta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/dailydelta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index delta1
+{
+        source                  = delta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/delta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+
+index tag1
+{
+        source                  = tag
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/tag1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+
+index note1
+{
+        source                  = note
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/note1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+
+indexer
+{
+        mem_limit               = 256M
+}
+
+
+searchd
+{
+        listen                  = 127.0.0.1:9312
+        listen                  = 127.0.0.1:9306:mysql41
+        log                     = /var/piler/manticore/manticore.log
+        binlog_path             = /var/piler/manticore/
+        query_log               = /var/piler/manticore/query.log
+        network_timeout         = 5
+        pid_file                = /var/run/piler/searchd.pid
+        seamless_rotate         = 1
+        preopen_indexes         = 1
+        unlink_old              = 1
+        thread_stack            = 512k
+}

init.d/rc.searchd.in

@@ -16,7 +16,13 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 
 NAME=searchd
 PID_FILE=/var/run/piler/searchd.pid
-PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
+PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE})
+CONFIG_FILE=sphinx.conf
+
+if [ -f SYSCONFDIR/piler/MANTICORE ]; then
+   CONFIG_FILE=manticore.conf
+fi
+
 
 start() {
 	echo "starting searchd . . ."
@@ -26,8 +32,8 @@ start() {
 		chown piler:piler /var/run/piler
 	fi
 
-        if [ `id -u` -eq 0 ]; then
+        if [ $(id -u) -eq 0 ]; then
-		su piler -c "searchd --config SYSCONFDIR/piler/sphinx.conf"
+		su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}"
 	else
 		searchd
 	fi
@@ -35,7 +41,7 @@ start() {
 
 stop() {
 	echo "stopping searchd"
-	kill ${PID_NUMBER}
+	kill "$PID_NUMBER"
 }
 
 check_status(){

util/indexer.delta.sh.in

@@ -6,10 +6,17 @@ DELTATMPFILE=/var/run/piler/delta.indexer.tmp
 INDEXER="indexer --config SYSCONFDIR/piler/sphinx.conf"
 PRIORITY=mail.info
 TOUCHFILE=/var/piler/stat/indexer
+CONFIG_FILE=sphinx.conf
 
-if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi
+if [[ -f SYSCONFDIR/piler/MANTICORE ]]; then
+   CONFIG_FILE=manticore.conf
+fi
 
-if [ -f $DELTATMPFILE ]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at "`cat $DELTATMPFILE` | logger -p $PRIORITY ; exit 1; fi
+INDEXER="indexer --config SYSCONFDIR/piler/${CONFIG_FILE}"
+
+if [[ -f "$MAINTMPFILE" ]]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at $(cat "$MAINTMPFILE")" | logger -p "$PRIORITY" ; exit 1; fi
+
+if [[ -f "$DELTATMPFILE" ]]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at $(cat "$DELTATMPFILE")" | logger -p "$PRIORITY" ; exit 1; fi
 
 date > $DELTATMPFILE
 
@@ -34,4 +41,3 @@ echo "INDEXER INFO: merging delta to dailydelta started" | logger -p $PRIORITY
 $INDEXER --quiet --merge dailydelta1 delta1 --merge-dst-range deleted 0 0 --rotate
 
 echo "INDEXER INFO: merging delta to dailydelta finished" | logger -p $PRIORITY
-

util/indexer.main.sh.in

@@ -5,11 +5,17 @@ set -o errexit
 set -o pipefail
 
 MAINTMPFILE="/var/run/piler/main.indexer.tmp"
-SPHINX_CONFIG="SYSCONFDIR/piler/sphinx.conf"
+CONFIG_FILE="SYSCONFDIR/piler/sphinx.conf"
+INDEXDIR=sphinx
 PRIORITY="mail.info"
 TOUCHFILE="/var/piler/stat/indexer"
 MAIN_INDEX="main1"
 
+if [[ -f SYSCONFDIR/piler/MANTICORE ]]; then
+   CONFIG_FILE=SYSCONFDIR/piler/manticore.conf
+   INDEXDIR=manticore
+fi
+
 export PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
 
 finish() {
@@ -30,7 +36,7 @@ trap finish EXIT
 
 echo "INDEXER INFO: merging to main started" | logger -p "$PRIORITY"
 
-indexer --config "$SPHINX_CONFIG" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate
+indexer --config "$CONFIG_FILE" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate
 
 echo "INDEXER INFO: merging to main finished" | logger -p "$PRIORITY"
 
@@ -38,12 +44,12 @@ sleep 5
 
 echo "INDEXER INFO: resetting daily delta started" | logger -p "$PRIORITY"
 
-indexer --config "$SPHINX_CONFIG" --quiet dailydelta1 --rotate
+indexer --config "$CONFIG_FILE" --quiet dailydelta1 --rotate
 
 echo "INDEXER INFO: resetting daily delta finished" | logger -p "$PRIORITY"
 
 sum=0
 while read -r a b; do
    sum=$(( sum + b ))
-done < <( find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" )
+done < <( find /var/piler/${INDEXDIR}/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" )
 printf "%d" $sum > /var/piler/stat/main_index_size

util/postinstall.sh.in

@@ -36,7 +36,7 @@ load_default_values() {
    MYSQL_ROOT_PASSWORD=""
    MYSQL_SOCKET=""
 
-   SPHINXCFG="${PILER_CONFIG_DIR}/sphinx.conf"
+   SEARCHCFG="${PILER_CONFIG_DIR}/manticore.conf"
 
    WWWGROUP="apache"
    DOCROOT="/var/piler/www"
@@ -196,9 +196,9 @@ askYN() {
 preinstall_check() {
    check_user root
 
-   if [[ "$INDEXER" == "" ]]; then "ERROR: cannot find sphinx indexer"; echo ""; exit ; fi
+   if [[ "$INDEXER" == "" ]]; then "ERROR: cannot find indexer binary"; echo ""; exit ; fi
 
-   if [[ "$SEARCHD" == "" ]]; then "ERROR: cannot find sphinx searchd"; echo ""; exit 0; fi
+   if [[ "$SEARCHD" == "" ]]; then "ERROR: cannot find searchd binary"; echo ""; exit 0; fi
 
    if [[ -f "$KEYFILE" ]]; then echo "ERROR: found existing keyfile (${KEYFILE}), aborting install"; echo ""; exit 0; fi
 }
@@ -281,8 +281,8 @@ make_cron_entries() {
       echo "30   2 * * * ${LIBEXECDIR}/piler/indexer.main.sh";
       echo "40 3 * * * ${LIBEXECDIR}/piler/purge.sh";
       echo "3 * * * * ${LIBEXECDIR}/piler/watch_sphinx_main_index.sh";
-      echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${PILER_CONFIG_DIR}/sphinx.conf";
+      echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${PILER_CONFIG_DIR}/manticore.conf";
-      echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${PILER_CONFIG_DIR}/sphinx.conf";
+      echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${PILER_CONFIG_DIR}/manticore.conf";
       echo "30   6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null";
       echo "*/5 * * * * /usr/bin/find ${LOCALSTATEDIR}/piler/error -type f|wc -l > ${LOCALSTATEDIR}/piler/stat/error";
       echo "*/5 * * * * /usr/bin/find ${DOCROOT}/tmp -type f -name i.\* -exec rm -f {} \;";
@@ -316,8 +316,8 @@ show_summary() {
    echo "mysql password: *******"
    echo
 
-   echo "sphinx indexer: ${INDEXER}"
+   echo "indexer: ${INDEXER}"
-   echo "sphinx config file: ${SPHINXCFG}"
+   echo "config file: ${SEARCHCFG}"
    echo
 
    echo "vhost docroot: ${DOCROOT}"
@@ -358,14 +358,12 @@ execute_post_install_tasks() {
    echo "Done."
 
    echo -n "Writing sphinx configuration... ";
-   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" -e "s%MYSQL_USERNAME%${MYSQL_USERNAME}%" -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" "${SYSCONFDIR}/piler/sphinx.conf.dist" > "$SPHINXCFG"
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" -e "s%MYSQL_USERNAME%${MYSQL_USERNAME}%" -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" "${SYSCONFDIR}/piler/manticore.conf.dist" > "$SEARCHCFG"
-   chmod +x "$SPHINXCFG"
+   chmod +x "$SEARCHCFG"
    echo "Done."
 
-   if indexer --version | grep ^Sphinx\ 3; then sed -i 's/220/311/' "$SPHINXCFG"; fi
+   echo -n "Initializing manticore indices... ";
-
+   su "$PILERUSER" -c "indexer --all --config ${SEARCHCFG}"
-   echo -n "Initializing sphinx indices... ";
-   su "$PILERUSER" -c "indexer --all --config ${SPHINXCFG}"
    echo "Done."