bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2025-01-12 14:30:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

accounting revision

Browse Source 
Change-Id: I6d808b4a2395b3d7400c2d7848c83b4b95748dbe
Signed-off-by: SJ <sj@acts.hu>
This commit is contained in:
SJ 2017-01-01 14:49:36 +01:00
parent 875b1150d1
commit 1722a3184b
2 changed files with 253 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
util/generate_stats.php
View File

@ -1,5 +1,5 @@
<?php <?php
// init/set default values
$webuidir = ''; $webuidir = '';
$process_all = false; $process_all = false;
$start = NULL; $start = NULL;
@ -8,45 +8,42 @@ $timestart = microtime(true);
ini_set("session.save_path", "/tmp"); ini_set("session.save_path", "/tmp");
// get options from command line
$opts = 'h::'; $opts = 'h::';
$lopts = array( $lopts = array(
'start:',
'stop:',
'webui:', 'webui:',
'start::',
'stop::',
); );
if ( $options = getopt( $opts, $lopts ) ) if($options = getopt($opts, $lopts)) {
{
if ( isset($options['webui']) )
{
$webuidir = $options['webui'];
} else
{
echo("\nError: must provide path to WebUI directory\n\n"); // todo: language
if(isset($options['h'])) {
display_help(); display_help();
exit;
} }
if ( isset($options['h']) ) if(isset($options['start'])) {
{
display_help();
exit;
}
if ( isset($options['start']) )
{
$start = $options['start']; $start = $options['start'];
} }
if ( isset($options['stop']) )
{ if(isset($options['stop'])) {
$stop = $options['stop']; $stop = $options['stop'];
} }
if(isset($options['webui'])) {
$webuidir = $options['webui'];
}
} else { } else {
display_help(); display_help();
exit;
} }
if(webuidir == '') {
echo("\nError: must provide path to WebUI directory\n\n");
display_help();
}
require_once($webuidir . "/config.php"); require_once($webuidir . "/config.php");
require(DIR_SYSTEM . "/startup.php"); require(DIR_SYSTEM . "/startup.php");
date_default_timezone_set(TIMEZONE); date_default_timezone_set(TIMEZONE);
@ -69,6 +66,7 @@ $db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_P
Registry::set('db', $db); Registry::set('db', $db);
$loader->model('accounting/accounting'); $loader->model('accounting/accounting');
$loader->model('domain/domain');
$messagestats = new ModelAccountingAccounting(); $messagestats = new ModelAccountingAccounting();
$_SESSION['username'] = 'cli-admin'; $_SESSION['username'] = 'cli-admin';
@ -76,7 +74,7 @@ $_SERVER['REMOTE_ADDR'] = '127.0.0.1';
extract($language->data); extract($language->data);
$records = $messagestats->run_counters($start,$stop); $records = $messagestats->run_counters($start, $stop, 'sent');
$timeend = microtime(true); $timeend = microtime(true);
$timegone = $timeend - $timestart; $timegone = $timeend - $timestart;
@ -89,7 +87,7 @@ echo("Removed ".$records['deletedstats']." records\n");
echo("Added ".$records['addedstats']." records\n"); echo("Added ".$records['addedstats']." records\n");
echo("Completed Run in ".$timegone." seconds\n\n"); echo("Completed Run in ".$timegone." seconds\n\n");
# Functions
function display_help() { function display_help() {
$phpself = basename(__FILE__); $phpself = basename(__FILE__);
echo("\nUsage: $phpself --webui [PATH] [OPTIONS...]\n\n"); echo("\nUsage: $phpself --webui [PATH] [OPTIONS...]\n\n");
@ -97,7 +95,8 @@ function display_help() {
echo("options:\n"); echo("options:\n");
echo("\t-a Reruns statistics for all records in the message view\n"); echo("\t-a Reruns statistics for all records in the message view\n");
echo("\t-h Prints this help screen and exits\n"); echo("\t-h Prints this help screen and exits\n");
echo("\t--start=\"Beginning of date range to process, ok values are today, yesterday or DDMMMYYYY...anything php's strtotime can process. Optional, will default to beginning of current day.\"\n"); echo("\t--start=\"Beginning of date range to process, ok values are today, yesterday or MYYYY/MM/DD...anything php's strtotime can process. Optional, will default to beginning of current day.\"\n");
echo("\t--stop=\"End of date range, same parameters as above. Optional (will default to end of current day)\"\n\n"); echo("\t--stop=\"End of date range, same parameters as above. Optional (will default to end of current day)\"\n\n");
exit;
} }
?>

187
webui/model/accounting/accounting.php
View File

@ -1,75 +1,69 @@
<?php <?php
class ModelAccountingAccounting extends Model class ModelAccountingAccounting extends Model {
{
public function run_counters( $start=NULL, $stop = NULL ) public function run_counters($start=NULL, $stop = NULL, $column = 'arrived') {
{
$now = time(); $now = time();
$counter = array(); $counter = array();
$accepteddomains = array_flip( $this->__getAcceptedDomains() ); $data = array(
$return = array(
'starttimestamp' => 0, 'starttimestamp' => 0,
'stoptimestamp' => 0, 'stoptimestamp' => 0,
'addedstats' => 0, 'addedstats' => 0,
'deletedstats' => 0, 'deletedstats' => 0
); );
if ( !is_null($start) ) if(!in_array($column, array('sent', 'arrived'))) { return $data; }
{
if(!is_null($start)) {
$start = $this->__decodeDate($start); $start = $this->__decodeDate($start);
} elseif ( is_null($start) ) } else {
{ //if we got nothing, then operate on today
//if we are passed nothing, operate on today
$start = $this->__decodeDate("00:00:00"); $start = $this->__decodeDate("00:00:00");
} }
if ( !is_null($stop) ) if(!is_null($stop)) {
{
$stop = $this->__decodeDate($stop); $stop = $this->__decodeDate($stop);
$stop = $stop + 86400; } else {
} elseif ( is_null($stop) ) //if we got nothing, then operate on today
{
//if we are passed nothing, operate on today
$stop = $this->__decodeDate("00:00:00"); $stop = $this->__decodeDate("00:00:00");
$stop = $stop + 86400;
} }
$stop += 86400;
$return['starttimestamp'] = $start;
$return['stoptimestamp'] = $stop;
// run query to return all messages $data['starttimestamp'] = $start;
$tousers = $this->db->query('SELECT ANY_VALUE(`sent`-(`sent`%86400)) as `day`,`to`,count(*) as `count`,sum(`size`) as `size` FROM ' . VIEW_MESSAGES . ' WHERE `sent` >= '.$start.' AND `sent` < '.$stop.' GROUP BY FROM_UNIXTIME(`day`, "%Y.%m.%d."), `to`;'); $data['stoptimestamp'] = $stop;
$fromusers = $this->db->query('SELECT ANY_VALUE(`sent`-(`sent`%86400)) as `day`,ANY_VALUE(`from`),count(*) as `count`,sum(`size`) as `size` FROM ' . VIEW_MESSAGES . ' WHERE `sent` >= '.$start.' AND `sent` < '.$stop.' GROUP BY FROM_UNIXTIME(`day`, "%Y.%m.%d."), `from`;');
// process results from above four queries
if($tousers->num_rows > 0) // emails sent to users
{
foreach($tousers->rows as $row) $tousers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `to`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `to`", array($start, $stop));
{
foreach($tousers->rows as $row) {
$counter[$row['day']][$row['to']]['recd'] = $row['count']; $counter[$row['day']][$row['to']]['recd'] = $row['count'];
$counter[$row['day']][$row['to']]['sizerecd'] = $row['size']; $counter[$row['day']][$row['to']]['sizerecd'] = $row['size'];
} }
}
if($fromusers->num_rows > 0)
{ // emails sent from users
foreach($fromusers->rows as $row)
{ $fromusers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `from`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `from`", array($start, $stop));
foreach($fromusers->rows as $row) {
$counter[$row['day']][$row['from']]['sent'] = $row['count']; $counter[$row['day']][$row['from']]['sent'] = $row['count'];
$counter[$row['day']][$row['from']]['sizesent'] = $row['size']; $counter[$row['day']][$row['from']]['sizesent'] = $row['size'];
} }
}
foreach( $counter as $date=>$users )
{ $accepteddomains = array_flip($this->model_domain_domain->get_mapped_domains());
foreach($counter as $date => $users) {
// execute queries to update the users and domains counter table // execute queries to update the users and domains counter table
$deletestats = $this->db->query("DELETE FROM " . TABLE_STAT_COUNTER . " WHERE `date` = $date;"); $deletestats = $this->db->query("DELETE FROM " . TABLE_STAT_COUNTER . " WHERE `date` = ?", array($date));
$return['deletedstats'] = $return['deletedstats'] + $this->db->countAffected(); $data['deletedstats'] += $this->db->countAffected();
foreach($users as $username => $userdata) {
foreach( $users as $username => $userdata)
{
//todo: consolidate
$sent = isset($userdata['sent']) ? $userdata['sent'] : 0; $sent = isset($userdata['sent']) ? $userdata['sent'] : 0;
$recd = isset($userdata['recd']) ? $userdata['recd'] : 0; $recd = isset($userdata['recd']) ? $userdata['recd'] : 0;
$sizesent = isset($userdata['sizesent']) ? $userdata['sizesent'] : 0; $sizesent = isset($userdata['sizesent']) ? $userdata['sizesent'] : 0;
@ -78,17 +72,17 @@ class ModelAccountingAccounting extends Model
$parts = explode('@', $username); $parts = explode('@', $username);
if(isset($parts[1]) && isset($accepteddomains[ $parts[1] ])) { if(isset($parts[1]) && isset($accepteddomains[ $parts[1] ])) {
$addusers = $this->db->query("INSERT INTO " . TABLE_STAT_COUNTER . " (`date`,`email`,`domain`,`sent`,`recd`,`sentsize`,`recdsize`) VALUES($date,'$username','".$parts[1]."',$sent,$recd,$sizesent,$sizerecd);"); $addusers = $this->db->query("INSERT INTO " . TABLE_STAT_COUNTER . " (`date`,`email`,`domain`,`sent`,`recd`,`sentsize`,`recdsize`) VALUES(?,?,?,?,?,?,?)", array($date, $username, $parts[1], $sent, $recd, $sizesent, $sizerecd));
$return['addedstats'] = $return['addedstats'] + $this->db->countAffected(); $data['addedstats'] += $this->db->countAffected();
}
} }
} }
if(LOG_LEVEL >= NORMAL) { syslog(LOG_INFO, sprintf("processed %s to %s: %d records deleted, %d records added", date(DATE_TEMPLATE, $data['starttimestamp']), date(DATE_TEMPLATE, $data['stoptimestamp']), $data['deletedstats'], $data['addedstats'])); }
return $data;
} }
if(LOG_LEVEL >= NORMAL) { syslog(LOG_INFO, sprintf("processed %s to %s: %d records deleted, %d records added",date(DATE_TEMPLATE, $return['starttimestamp']),date(DATE_TEMPLATE, $return['stoptimestamp']),$return['deletedstats'],$return['addedstats'])); }
return $return;
}
public function get_accounting($item='email', $search='', $page=0, $pagelen=0, $sort='item', $order=0) { public function get_accounting($item='email', $search='', $page=0, $pagelen=0, $sort='item', $order=0) {
@ -97,21 +91,6 @@ class ModelAccountingAccounting extends Model
$_order = 0; $_order = 0;
$_order = ""; $_order = "";
$limit = ""; $limit = "";
$return = array();
/*
if(MEMCACHED_ENABLED) {
$memcache = Registry::get('memcache');
$statscounter = $memcache->get(Registry::get('statscounters'));
if(isset($counter[MEMCACHED_PREFIX . 'counters_last_update'])) {
if(isset($counter[MEMCACHED_PREFIX . 'size'])) { $asize = nice_size($counter[MEMCACHED_PREFIX . 'size'], ' '); }
unset($counter[MEMCACHED_PREFIX . 'size']);
return array ($asize, $counter);
}
} */
$account_for_emails = $this->__getEmails(); $account_for_emails = $this->__getEmails();
$account_for_domains = $this->__getDomains(); $account_for_domains = $this->__getDomains();
@ -150,8 +129,7 @@ class ModelAccountingAccounting extends Model
$query = $this->db->query($query.' '.$where.' '.$group.' '.$_order.' '.$limit.';'); $query = $this->db->query($query.' '.$where.' '.$group.' '.$_order.' '.$limit.';');
if($query->num_rows >= 1) if($query->num_rows >= 1) {
{
return $query->rows; return $query->rows;
} else { } else {
// no results found // no results found
@ -161,107 +139,90 @@ class ModelAccountingAccounting extends Model
public function count_accounting($item='email', $search='') { public function count_accounting($item='email', $search='') {
$account_for_emails = $this->__getEmails(); $account_for_emails = $this->__getEmails();
$account_for_domains = $this->__getDomains(); $account_for_domains = $this->__getDomains();
$search = preg_replace("/\s{1,}/", "", $search); $search = preg_replace("/\s{1,}/", "", $search);
if($search) { if($search) {
$search_cond .= " AND ( `email` like '%".$search."%' OR `domain` like '%".$search."%' )"; $search_cond .= " AND ( `email` LIKE '%".$search."%' OR `domain` LIKE '%".$search."%' )";
} }
$query = "SELECT `email` AS `item`,MIN(`date`) as `oldest`,MAX(`date`) as `newest`,sum(`sent`) as `sent`,sum(`recd`) as `recd`,sum(`sentsize`) as `sentsize`,sum(`recdsize`) as `recdsize` FROM " . TABLE_STAT_COUNTER; $query = "SELECT `email` AS `item`, MIN(`date`) AS `oldest`, MAX(`date`) AS `newest`, SUM(`sent`) AS `sent`, SUM(`recd`) AS `recd`, SUM(`sentsize`) AS `sentsize`, SUM(`recdsize`) AS `recdsize` FROM " . TABLE_STAT_COUNTER;
if($item == 'email') { if($item == 'email') {
$where = "WHERE `email` IN ('".implode("','",$account_for_emails)."') OR `domain` IN ('".implode("','",$account_for_domains)."')"; $where = "WHERE `email` IN ('".implode("','",$account_for_emails)."') OR `domain` IN ('".implode("','",$account_for_domains)."')";
if($search) { if($search) {
$where .= " AND ( `email` like '%".$search."%' OR `domain` like '%".$search."%' )"; $where .= " AND ( `email` LIKE '%".$search."%' OR `domain` LIKE '%".$search."%' )";
} }
$group = "GROUP BY `email`"; $group = "GROUP BY `email`";
} elseif ($item == 'domain') { } elseif ($item == 'domain') {
$where = "WHERE `domain` IN ('".implode("','",$account_for_domains)."')"; $where = "WHERE `domain` IN ('".implode("','",$account_for_domains)."')";
if($search) { if($search) {
$where .= " AND `domain` like '%".$search."%'"; $where .= " AND `domain` LIKE '%".$search."%'";
} }
$group = "GROUP BY `domain`"; $group = "GROUP BY `domain`";
} else { } else {
return false; return false;
} }
$query = $this->db->query($query.' '.$where.' '.$group.';'); $query = $this->db->query($query.' '.$where.' '.$group);
return $query->num_rows; return $query->num_rows;
} }
private function __getEmails() { private function __getEmails() {
$return = array(); $emails = array();
$session = Registry::get('session'); $session = Registry::get('session');
array_push($return, $session->get("email")); array_push($emails, $session->get("email"));
$emails = $session->get("emails"); $__emails = $session->get("emails");
foreach ($emails as $e) { foreach ($__emails as $e) {
array_push($return,$e); array_push($emails, $e);
} }
return $return; return $emails;
} }
private function __getDomains() { private function __getDomains() {
$return = array(); $domains = array();
$session = Registry::get('session'); $session = Registry::get('session');
if(Registry::get('admin_user') >= 1) { if(Registry::get('admin_user') >= 1) {
$return = $this->__getAcceptedDomains(); $domains = $this->model_domain_domain->get_mapped_domains();
}elseif(Registry::get('auditor_user') == 1) { }
array_push($return, $session->get("domain")); elseif(Registry::get('auditor_user') == 1) {
array_push($domains, $session->get("domain"));
$auditdomains = $session->get("auditdomains"); $auditdomains = $session->get("auditdomains");
foreach ($auditdomains as $d) { foreach ($auditdomains as $d) {
array_push($return,$d); array_push($domains, $d);
} }
} }
return $return; return $domains;
} }
private function __getAcceptedDomains() {
// todo: move to domains model?
$return = array();
$query = $this->db->query("SELECT domain, mapped FROM " . TABLE_DOMAIN . " ORDER BY domain ASC");
foreach($query->rows as $domain)
{
if ($domain['domain'] == $domain['mapped']) {
array_push($return,$domain['domain']);
} else {
array_push($return,$domain['mapped']);
}
}
return $return;
}
private function __decodeDate($input) { private function __decodeDate($input) {
$timestamp = 0;
if ( !is_numeric($input) ) if(!is_numeric($input)) {
{ // if we got anything, but a timestamp, eg. words (yesterday, today)
// if we are passed words (yesterday, today), convert to unix timestamps representing the start of the day // then convert to unix timestamp representing the start of the day
$input = strtotime($input); $input = strtotime($input);
$return = $input - ($input%86400); $timestamp = $input - ($input % 86400);
}
} elseif ( is_numeric($input) ) else {
{ // round the timestamp to the start of the day
// if we are passed unix timestamps, ensure they represent the start of the day $timestamp = $input - ($input % 86400);
$return = $input - ($input%86400);
} }
return $return; return $timestamp;
} }
} }
?>