Fixed #758: sphinx index watcher

Signed-off-by: Janos SUTO <sj@acts.hu>
This commit is contained in:
Janos SUTO 2018-03-18 13:54:13 +01:00
parent c918ad2a66
commit c12a876e59
9 changed files with 28 additions and 9 deletions

View File

@ -171,6 +171,8 @@ $config['PAGE_LEN'] = 20;
$config['MAX_NUMBER_OF_FROM_ITEMS'] = 5; $config['MAX_NUMBER_OF_FROM_ITEMS'] = 5;
$config['MAX_SEARCH_HITS'] = 1000; $config['MAX_SEARCH_HITS'] = 1000;
$config['SPHINX_MAIN_INDEX_THRESHOLD'] = 2000000000;
$config['DEFAULT_RETENTION'] = 0; $config['DEFAULT_RETENTION'] = 0;
$config['LOCALHOST'] = '127.0.0.1'; $config['LOCALHOST'] = '127.0.0.1';
@ -419,6 +421,7 @@ define('QSHAPE_DEFERRED_SENDER', DIR_STAT . '/deferred-sender');
define('CPUSTAT', DIR_STAT . '/cpu.stat'); define('CPUSTAT', DIR_STAT . '/cpu.stat');
define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat'); define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat');
define('ARCHIVE_SIZE', DIR_STAT . '/archive.size'); define('ARCHIVE_SIZE', DIR_STAT . '/archive.size');
define('SPHINX_MAIN_INDEX_SIZE', DIR_STAT . '/main_index_size');
define('LOCK_FILE', DIR_LOG . 'lock'); define('LOCK_FILE', DIR_LOG . 'lock');
define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php'); define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php');

View File

@ -1,11 +1,11 @@
### PILERSTART ### PILERSTART
5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh 5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
30 2 * * * LIBEXECDIR/piler/indexer.main.sh 30 2 * * * LIBEXECDIR/piler/indexer.main.sh
15,45 * * * * LIBEXECDIR/piler/indexer.attachment.sh
*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate */15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate
*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate */15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \; */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
3 * * * * LIBEXECDIR/piler/watch_sphinx_main_index.sh
### optional: the same report you can see on the health page ### optional: the same report you can see on the health page
###30 7 * * * /usr/bin/php LIBEXECDIR/piler/daily-report.php --webui LOCALSTATEDIR/piler/www ###30 7 * * * /usr/bin/php LIBEXECDIR/piler/daily-report.php --webui LOCALSTATEDIR/piler/www

View File

@ -44,6 +44,7 @@ install:
$(INSTALL) -m 0755 $(srcdir)/purge.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/purge.sh $(DESTDIR)$(libexecdir)/piler
$(INSTALL) -m 0755 $(srcdir)/pilerpurge.py $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/pilerpurge.py $(DESTDIR)$(libexecdir)/piler
$(INSTALL) -m 0755 $(srcdir)/postinstall.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/postinstall.sh $(DESTDIR)$(libexecdir)/piler
$(INSTALL) -m 0755 $(srcdir)/watch_sphinx_main_index.sh $(DESTDIR)$(libexecdir)/piler
$(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler
$(INSTALL) -m 0755 $(srcdir)/db-mysql-root.sql.in $(DESTDIR)$(datarootdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql-root.sql.in $(DESTDIR)$(datarootdir)/piler

View File

@ -277,7 +277,7 @@ make_cron_entries() {
echo "### PILERSTART" >> "$CRON_TMP" echo "### PILERSTART" >> "$CRON_TMP"
echo "5,35 * * * * ${LIBEXECDIR}/piler/indexer.delta.sh" >> "$CRON_TMP" echo "5,35 * * * * ${LIBEXECDIR}/piler/indexer.delta.sh" >> "$CRON_TMP"
echo "30 2 * * * ${LIBEXECDIR}/piler/indexer.main.sh" >> "$CRON_TMP" echo "30 2 * * * ${LIBEXECDIR}/piler/indexer.main.sh" >> "$CRON_TMP"
echo "15,45 * * * * ${LIBEXECDIR}/piler/indexer.attachment.sh" >> "$CRON_TMP" echo "3 * * * * ${LIBEXECDIR}/watch_sphinx_main_index.sh" >> "$CRON_TMP"
echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${SYSCONFDIR}/piler/sphinx.conf" >> "$CRON_TMP" echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${SYSCONFDIR}/piler/sphinx.conf" >> "$CRON_TMP"
echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${SYSCONFDIR}/piler/sphinx.conf" >> "$CRON_TMP" echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${SYSCONFDIR}/piler/sphinx.conf" >> "$CRON_TMP"
echo "30 6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null" >> "$CRON_TMP" echo "30 6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null" >> "$CRON_TMP"

View File

@ -0,0 +1,3 @@
#!/bin/bash
find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" | sort -r | head -1 | cut -f2 -d ' ' > /var/piler/stat/main_index_size

View File

@ -156,11 +156,10 @@ class ControllerHealthWorker extends Controller {
$this->data['indexer_stat'] = $this->model_health_health->indexer_stat(); $this->data['indexer_stat'] = $this->model_health_health->indexer_stat();
$this->data['purge_stat'] = $this->model_health_health->purge_stat(); $this->data['purge_stat'] = $this->model_health_health->purge_stat();
$this->data['sphinx_current_main_size'] = $this->model_health_health->get_current_sphinx_main_index_size();
$this->render(); $this->render();
} }
} }
?>

View File

@ -255,7 +255,15 @@ class ModelHealthHealth extends Model {
return $data; return $data;
} }
public function get_current_sphinx_main_index_size() {
$size = 0;
if(file_exists(SPHINX_MAIN_INDEX_SIZE)) {
$size = (int) file_get_contents(SPHINX_MAIN_INDEX_SIZE);
}
return $size;
}
} }
?>

View File

@ -115,8 +115,8 @@ cite{font-style:normal;}
a.muted:hover{color:#808080;} a.muted:hover{color:#808080;}
.text-warning{color:#c09853;} .text-warning{color:#c09853;}
a.text-warning:hover{color:#a47e3c;} a.text-warning:hover{color:#a47e3c;}
.text-error{color:#b94a48;} .text-error{color:#b94a48;font-weight:bold;}
a.text-error:hover{color:#953b39;} a.text-error:hover{color:#953b39;font-weight:bold;}
.text-info{color:#3a87ad;} .text-info{color:#3a87ad;}
a.text-info:hover{color:#2d6987;} a.text-info:hover{color:#2d6987;}
.text-success{color:#468847;} .text-success{color:#468847;}

View File

@ -61,6 +61,7 @@
</tr> </tr>
<?php } ?> <?php } ?>
</table> </table>
</td> </td>
</tr> </tr>
@ -154,6 +155,10 @@
<td><?php print $text_usage_trend; ?></td> <td><?php print $text_usage_trend; ?></td>
<td><?php if ( $usagetrend > 0 ) { print $text_usage_increasing; } elseif( $usagetrend < 0 ) { print $text_usage_decreasing; } else { print $text_usage_neutral; } ?></td> <td><?php if ( $usagetrend > 0 ) { print $text_usage_increasing; } elseif( $usagetrend < 0 ) { print $text_usage_decreasing; } else { print $text_usage_neutral; } ?></td>
</tr> </tr>
<tr>
<td>Sphinx main index</td>
<td<?php if($sphinx_current_main_size > SPHINX_MAIN_INDEX_THRESHOLD) { ?> class="text-error"<?php } ?>><?php print nice_size($sphinx_current_main_size); ?></td>
</tr>
</table> </table>