From c12a876e5983d48070ad38dd5fb01dadd53714ef Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 18 Mar 2018 13:54:13 +0100 Subject: [PATCH] Fixed #758: sphinx index watcher Signed-off-by: Janos SUTO --- config.php.in | 3 +++ etc/cron.jobs.in | 2 +- util/Makefile.in | 1 + util/postinstall.sh.in | 2 +- util/watch_sphinx_main_index.sh | 3 +++ webui/controller/health/worker.php | 3 +-- webui/model/health/health.php | 14 +++++++++++--- .../theme/default/assets/css/metro-bootstrap.css | 4 ++-- .../view/theme/default/templates/health/worker.tpl | 5 +++++ 9 files changed, 28 insertions(+), 9 deletions(-) create mode 100755 util/watch_sphinx_main_index.sh diff --git a/config.php.in b/config.php.in index 4a3fa914..a336d2a7 100644 --- a/config.php.in +++ b/config.php.in @@ -171,6 +171,8 @@ $config['PAGE_LEN'] = 20; $config['MAX_NUMBER_OF_FROM_ITEMS'] = 5; $config['MAX_SEARCH_HITS'] = 1000; +$config['SPHINX_MAIN_INDEX_THRESHOLD'] = 2000000000; + $config['DEFAULT_RETENTION'] = 0; $config['LOCALHOST'] = '127.0.0.1'; @@ -419,6 +421,7 @@ define('QSHAPE_DEFERRED_SENDER', DIR_STAT . '/deferred-sender'); define('CPUSTAT', DIR_STAT . '/cpu.stat'); define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat'); define('ARCHIVE_SIZE', DIR_STAT . '/archive.size'); +define('SPHINX_MAIN_INDEX_SIZE', DIR_STAT . '/main_index_size'); define('LOCK_FILE', DIR_LOG . 'lock'); define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php'); diff --git a/etc/cron.jobs.in b/etc/cron.jobs.in index 40377b2f..3604671c 100644 --- a/etc/cron.jobs.in +++ b/etc/cron.jobs.in @@ -1,11 +1,11 @@ ### PILERSTART 5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh 30 2 * * * LIBEXECDIR/piler/indexer.main.sh -15,45 * * * * LIBEXECDIR/piler/indexer.attachment.sh */15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate */15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \; */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error +3 * * * * LIBEXECDIR/piler/watch_sphinx_main_index.sh ### optional: the same report you can see on the health page ###30 7 * * * /usr/bin/php LIBEXECDIR/piler/daily-report.php --webui LOCALSTATEDIR/piler/www diff --git a/util/Makefile.in b/util/Makefile.in index a0be994f..31deefa8 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -44,6 +44,7 @@ install: $(INSTALL) -m 0755 $(srcdir)/purge.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/pilerpurge.py $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/postinstall.sh $(DESTDIR)$(libexecdir)/piler + $(INSTALL) -m 0755 $(srcdir)/watch_sphinx_main_index.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql-root.sql.in $(DESTDIR)$(datarootdir)/piler diff --git a/util/postinstall.sh.in b/util/postinstall.sh.in index d0d028ec..22ec17c8 100755 --- a/util/postinstall.sh.in +++ b/util/postinstall.sh.in @@ -277,7 +277,7 @@ make_cron_entries() { echo "### PILERSTART" >> "$CRON_TMP" echo "5,35 * * * * ${LIBEXECDIR}/piler/indexer.delta.sh" >> "$CRON_TMP" echo "30 2 * * * ${LIBEXECDIR}/piler/indexer.main.sh" >> "$CRON_TMP" - echo "15,45 * * * * ${LIBEXECDIR}/piler/indexer.attachment.sh" >> "$CRON_TMP" + echo "3 * * * * ${LIBEXECDIR}/watch_sphinx_main_index.sh" >> "$CRON_TMP" echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${SYSCONFDIR}/piler/sphinx.conf" >> "$CRON_TMP" echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${SYSCONFDIR}/piler/sphinx.conf" >> "$CRON_TMP" echo "30 6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null" >> "$CRON_TMP" diff --git a/util/watch_sphinx_main_index.sh b/util/watch_sphinx_main_index.sh new file mode 100755 index 00000000..3d330f95 --- /dev/null +++ b/util/watch_sphinx_main_index.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" | sort -r | head -1 | cut -f2 -d ' ' > /var/piler/stat/main_index_size diff --git a/webui/controller/health/worker.php b/webui/controller/health/worker.php index d32a5bf9..3e68531b 100644 --- a/webui/controller/health/worker.php +++ b/webui/controller/health/worker.php @@ -156,11 +156,10 @@ class ControllerHealthWorker extends Controller { $this->data['indexer_stat'] = $this->model_health_health->indexer_stat(); $this->data['purge_stat'] = $this->model_health_health->purge_stat(); + $this->data['sphinx_current_main_size'] = $this->model_health_health->get_current_sphinx_main_index_size(); $this->render(); } } - -?> diff --git a/webui/model/health/health.php b/webui/model/health/health.php index 459a6418..738cdf76 100644 --- a/webui/model/health/health.php +++ b/webui/model/health/health.php @@ -255,7 +255,15 @@ class ModelHealthHealth extends Model { return $data; } + + public function get_current_sphinx_main_index_size() { + $size = 0; + + if(file_exists(SPHINX_MAIN_INDEX_SIZE)) { + $size = (int) file_get_contents(SPHINX_MAIN_INDEX_SIZE); + } + + return $size; + } + } - - -?> diff --git a/webui/view/theme/default/assets/css/metro-bootstrap.css b/webui/view/theme/default/assets/css/metro-bootstrap.css index 8e8be126..1f639103 100644 --- a/webui/view/theme/default/assets/css/metro-bootstrap.css +++ b/webui/view/theme/default/assets/css/metro-bootstrap.css @@ -115,8 +115,8 @@ cite{font-style:normal;} a.muted:hover{color:#808080;} .text-warning{color:#c09853;} a.text-warning:hover{color:#a47e3c;} -.text-error{color:#b94a48;} -a.text-error:hover{color:#953b39;} +.text-error{color:#b94a48;font-weight:bold;} +a.text-error:hover{color:#953b39;font-weight:bold;} .text-info{color:#3a87ad;} a.text-info:hover{color:#2d6987;} .text-success{color:#468847;} diff --git a/webui/view/theme/default/templates/health/worker.tpl b/webui/view/theme/default/templates/health/worker.tpl index 7ce8db0c..0a20c4e4 100644 --- a/webui/view/theme/default/templates/health/worker.tpl +++ b/webui/view/theme/default/templates/health/worker.tpl @@ -61,6 +61,7 @@ + @@ -154,6 +155,10 @@ 0 ) { print $text_usage_increasing; } elseif( $usagetrend < 0 ) { print $text_usage_decreasing; } else { print $text_usage_neutral; } ?> + + Sphinx main index + SPHINX_MAIN_INDEX_THRESHOLD) { ?> class="text-error"> +