accounting revision

Change-Id: I6d808b4a2395b3d7400c2d7848c83b4b95748dbe
Signed-off-by: SJ <sj@acts.hu>
This commit is contained in:
SJ 2017-01-01 14:49:36 +01:00
parent 875b1150d1
commit 1722a3184b
2 changed files with 253 additions and 293 deletions

View File

@ -1,5 +1,5 @@
<?php
// init/set default values
$webuidir = '';
$process_all = false;
$start = NULL;
@ -8,45 +8,42 @@ $timestart = microtime(true);
ini_set("session.save_path", "/tmp");
// get options from command line
$opts = 'h::';
$lopts = array(
'webui:',
'start::',
'stop::',
);
'start:',
'stop:',
'webui:',
);
if ( $options = getopt( $opts, $lopts ) )
{
if ( isset($options['webui']) )
{
$webuidir = $options['webui'];
} else
{
echo("\nError: must provide path to WebUI directory\n\n"); // todo: language
if($options = getopt($opts, $lopts)) {
display_help();
exit;
}
if(isset($options['h'])) {
display_help();
}
if(isset($options['start'])) {
$start = $options['start'];
}
if(isset($options['stop'])) {
$stop = $options['stop'];
}
if(isset($options['webui'])) {
$webuidir = $options['webui'];
}
if ( isset($options['h']) )
{
display_help();
exit;
}
if ( isset($options['start']) )
{
$start = $options['start'];
}
if ( isset($options['stop']) )
{
$stop = $options['stop'];
}
} else {
display_help();
exit;
}
if(webuidir == '') {
echo("\nError: must provide path to WebUI directory\n\n");
display_help();
}
require_once($webuidir . "/config.php");
require(DIR_SYSTEM . "/startup.php");
date_default_timezone_set(TIMEZONE);
@ -69,6 +66,7 @@ $db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_P
Registry::set('db', $db);
$loader->model('accounting/accounting');
$loader->model('domain/domain');
$messagestats = new ModelAccountingAccounting();
$_SESSION['username'] = 'cli-admin';
@ -76,7 +74,7 @@ $_SERVER['REMOTE_ADDR'] = '127.0.0.1';
extract($language->data);
$records = $messagestats->run_counters($start,$stop);
$records = $messagestats->run_counters($start, $stop, 'sent');
$timeend = microtime(true);
$timegone = $timeend - $timestart;
@ -89,15 +87,16 @@ echo("Removed ".$records['deletedstats']." records\n");
echo("Added ".$records['addedstats']." records\n");
echo("Completed Run in ".$timegone." seconds\n\n");
# Functions
function display_help() {
$phpself = basename(__FILE__);
echo("\nUsage: $phpself --webui [PATH] [OPTIONS...]\n\n");
echo("\t--webui=\"[REQUIRED: path to the Piler WebUI Directory]\"\n\n");
echo("options:\n");
echo("\t-a Reruns statistics for all records in the message view\n");
echo("\t-h Prints this help screen and exits\n");
echo("\t--start=\"Beginning of date range to process, ok values are today, yesterday or DDMMMYYYY...anything php's strtotime can process. Optional, will default to beginning of current day.\"\n");
echo("\t--stop=\"End of date range, same parameters as above. Optional (will default to end of current day)\"\n\n");
$phpself = basename(__FILE__);
echo("\nUsage: $phpself --webui [PATH] [OPTIONS...]\n\n");
echo("\t--webui=\"[REQUIRED: path to the Piler WebUI Directory]\"\n\n");
echo("options:\n");
echo("\t-a Reruns statistics for all records in the message view\n");
echo("\t-h Prints this help screen and exits\n");
echo("\t--start=\"Beginning of date range to process, ok values are today, yesterday or MYYYY/MM/DD...anything php's strtotime can process. Optional, will default to beginning of current day.\"\n");
echo("\t--stop=\"End of date range, same parameters as above. Optional (will default to end of current day)\"\n\n");
exit;
}
?>

View File

@ -1,267 +1,228 @@
<?php
class ModelAccountingAccounting extends Model
{
class ModelAccountingAccounting extends Model {
public function run_counters( $start=NULL, $stop = NULL )
{
$now = time();
$counter = array();
$accepteddomains = array_flip( $this->__getAcceptedDomains() );
$return = array(
'starttimestamp' => 0,
'stoptimestamp' => 0,
'addedstats' => 0,
'deletedstats' => 0,
);
public function run_counters($start=NULL, $stop = NULL, $column = 'arrived') {
$now = time();
$counter = array();
$data = array(
'starttimestamp' => 0,
'stoptimestamp' => 0,
'addedstats' => 0,
'deletedstats' => 0
);
if ( !is_null($start) )
{
$start = $this->__decodeDate( $start );
} elseif ( is_null($start) )
{
//if we are passed nothing, operate on today
$start = $this->__decodeDate( "00:00:00" );
}
if(!in_array($column, array('sent', 'arrived'))) { return $data; }
if ( !is_null($stop) )
{
$stop = $this->__decodeDate( $stop );
$stop = $stop + 86400;
} elseif ( is_null($stop) )
{
//if we are passed nothing, operate on today
$stop = $this->__decodeDate( "00:00:00" );
$stop = $stop + 86400;
}
if(!is_null($start)) {
$start = $this->__decodeDate($start);
} else {
//if we got nothing, then operate on today
$start = $this->__decodeDate("00:00:00");
}
$return['starttimestamp'] = $start;
$return['stoptimestamp'] = $stop;
if(!is_null($stop)) {
$stop = $this->__decodeDate($stop);
} else {
//if we got nothing, then operate on today
$stop = $this->__decodeDate("00:00:00");
}
$stop += 86400;
// run query to return all messages
$tousers = $this->db->query('SELECT ANY_VALUE(`sent`-(`sent`%86400)) as `day`,`to`,count(*) as `count`,sum(`size`) as `size` FROM ' . VIEW_MESSAGES . ' WHERE `sent` >= '.$start.' AND `sent` < '.$stop.' GROUP BY FROM_UNIXTIME(`day`, "%Y.%m.%d."), `to`;');
$fromusers = $this->db->query('SELECT ANY_VALUE(`sent`-(`sent`%86400)) as `day`,ANY_VALUE(`from`),count(*) as `count`,sum(`size`) as `size` FROM ' . VIEW_MESSAGES . ' WHERE `sent` >= '.$start.' AND `sent` < '.$stop.' GROUP BY FROM_UNIXTIME(`day`, "%Y.%m.%d."), `from`;');
// process results from above four queries
if($tousers->num_rows > 0)
{
foreach($tousers->rows as $row)
{
$counter[$row['day']][$row['to']]['recd'] = $row['count'];
$counter[$row['day']][$row['to']]['sizerecd'] = $row['size'];
$data['starttimestamp'] = $start;
$data['stoptimestamp'] = $stop;
// emails sent to users
$tousers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `to`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `to`", array($start, $stop));
foreach($tousers->rows as $row) {
$counter[$row['day']][$row['to']]['recd'] = $row['count'];
$counter[$row['day']][$row['to']]['sizerecd'] = $row['size'];
}
// emails sent from users
$fromusers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `from`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `from`", array($start, $stop));
foreach($fromusers->rows as $row) {
$counter[$row['day']][$row['from']]['sent'] = $row['count'];
$counter[$row['day']][$row['from']]['sizesent'] = $row['size'];
}
$accepteddomains = array_flip($this->model_domain_domain->get_mapped_domains());
foreach($counter as $date => $users) {
// execute queries to update the users and domains counter table
$deletestats = $this->db->query("DELETE FROM " . TABLE_STAT_COUNTER . " WHERE `date` = ?", array($date));
$data['deletedstats'] += $this->db->countAffected();
foreach($users as $username => $userdata) {
$sent = isset($userdata['sent']) ? $userdata['sent'] : 0;
$recd = isset($userdata['recd']) ? $userdata['recd'] : 0;
$sizesent = isset($userdata['sizesent']) ? $userdata['sizesent'] : 0;
$sizerecd = isset($userdata['sizerecd']) ? $userdata['sizerecd'] : 0;
$parts = explode('@', $username);
if(isset($parts[1]) && isset($accepteddomains[ $parts[1] ])) {
$addusers = $this->db->query("INSERT INTO " . TABLE_STAT_COUNTER . " (`date`,`email`,`domain`,`sent`,`recd`,`sentsize`,`recdsize`) VALUES(?,?,?,?,?,?,?)", array($date, $username, $parts[1], $sent, $recd, $sizesent, $sizerecd));
$data['addedstats'] += $this->db->countAffected();
}
}
if($fromusers->num_rows > 0)
{
foreach($fromusers->rows as $row)
{
$counter[$row['day']][$row['from']]['sent'] = $row['count'];
$counter[$row['day']][$row['from']]['sizesent'] = $row['size'];
}
}
foreach( $counter as $date=>$users )
{
// execute queries to update the users and domains counter table
$deletestats = $this->db->query("DELETE FROM " . TABLE_STAT_COUNTER . " WHERE `date` = $date;");
$return['deletedstats'] = $return['deletedstats'] + $this->db->countAffected();
foreach( $users as $username => $userdata)
{
//todo: consolidate
$sent = isset($userdata['sent']) ? $userdata['sent'] : 0;
$recd = isset($userdata['recd']) ? $userdata['recd'] : 0;
$sizesent = isset($userdata['sizesent']) ? $userdata['sizesent'] : 0;
$sizerecd = isset($userdata['sizerecd']) ? $userdata['sizerecd'] : 0;
$parts = explode('@',$username);
if ( isset($parts[1]) && isset($accepteddomains[ $parts[1] ]) ) {
$addusers = $this->db->query("INSERT INTO " . TABLE_STAT_COUNTER . " (`date`,`email`,`domain`,`sent`,`recd`,`sentsize`,`recdsize`) VALUES($date,'$username','".$parts[1]."',$sent,$recd,$sizesent,$sizerecd);");
$return['addedstats'] = $return['addedstats'] + $this->db->countAffected();
}
}
}
if(LOG_LEVEL >= NORMAL) { syslog(LOG_INFO, sprintf("processed %s to %s: %d records deleted, %d records added",date(DATE_TEMPLATE, $return['starttimestamp']),date(DATE_TEMPLATE, $return['stoptimestamp']),$return['deletedstats'],$return['addedstats'])); }
return $return;
}
public function get_accounting($item = 'email',$search='',$page=0,$pagelen=0,$sort='item',$order=0 ) {
// item can be either email or domain, maybe folder in the future??
$_order = 0;
$_order = "";
$limit = "";
$return = array();
/*
if(MEMCACHED_ENABLED) {
$memcache = Registry::get('memcache');
$statscounter = $memcache->get(Registry::get('statscounters'));
if(isset($counter[MEMCACHED_PREFIX . 'counters_last_update'])) {
if(isset($counter[MEMCACHED_PREFIX . 'size'])) { $asize = nice_size($counter[MEMCACHED_PREFIX . 'size'], ' '); }
unset($counter[MEMCACHED_PREFIX . 'size']);
return array ($asize, $counter);
}
} */
}
$account_for_emails = $this->__getEmails();
$account_for_domains = $this->__getDomains();
if(LOG_LEVEL >= NORMAL) { syslog(LOG_INFO, sprintf("processed %s to %s: %d records deleted, %d records added", date(DATE_TEMPLATE, $data['starttimestamp']), date(DATE_TEMPLATE, $data['stoptimestamp']), $data['deletedstats'], $data['addedstats'])); }
$search = preg_replace("/\s{1,}/", "", $search);
return $data;
}
if ($item == 'email') {
$account_for_emails = $this->__getEmails();
$account_for_domains = $this->__getDomains();
$query = "SELECT `email` AS `item`,MIN(`date`) as `oldest`,MAX(`date`) as `newest`,sum(`sent`) as `sent`,sum(`recd`) as `recd`,SUM(`sentsize`) as `sentsize`,AVG(`sentsize`) as `sentavg`,SUM(`recdsize`) as `recdsize`,AVG(`recdsize`) as `recdavg` FROM " . TABLE_STAT_COUNTER;
$where = "WHERE ( `email` IN ('".implode("','",$account_for_emails)."') OR `domain` IN ('".implode("','",$account_for_domains)."') )";
if($search){
$where .= " AND ( `email` like '%".$search."%' OR `domain` like '%".$search."%' )";
}
$group = "GROUP BY `email`";
} elseif ($item == 'domain') {
$account_for_domains = $this->__getDomains();
$query = "SELECT `domain` AS `item`,MIN(`date`) as `oldest`,MAX(`date`) as `newest`,sum(`sent`) as `sent`,sum(`recd`) as `recd`,SUM(`sentsize`) as `sentsize`,AVG(`sentsize`) as `sentavg`,SUM(`recdsize`) as `recdsize`,AVG(`recdsize`) as `recdavg` FROM " . TABLE_STAT_COUNTER;
$where = "WHERE ( `domain` IN ('".implode("','",$account_for_domains)."') )";
if($search){
$where .= " AND `domain` like '%".$search."%'";
}
$group = "GROUP BY `domain`";
} else {
return false;
public function get_accounting($item='email', $search='', $page=0, $pagelen=0, $sort='item', $order=0) {
// item can be either email or domain, maybe folder in the future??
$_order = 0;
$_order = "";
$limit = "";
$account_for_emails = $this->__getEmails();
$account_for_domains = $this->__getDomains();
$search = preg_replace("/\s{1,}/", "", $search);
if($item == 'email') {
$account_for_emails = $this->__getEmails();
$account_for_domains = $this->__getDomains();
$query = "SELECT `email` AS `item`,MIN(`date`) as `oldest`,MAX(`date`) as `newest`,sum(`sent`) as `sent`,sum(`recd`) as `recd`,SUM(`sentsize`) as `sentsize`,AVG(`sentsize`) as `sentavg`,SUM(`recdsize`) as `recdsize`,AVG(`recdsize`) as `recdavg` FROM " . TABLE_STAT_COUNTER;
$where = "WHERE ( `email` IN ('".implode("','",$account_for_emails)."') OR `domain` IN ('".implode("','",$account_for_domains)."') )";
if($search) {
$where .= " AND ( `email` like '%".$search."%' OR `domain` like '%".$search."%' )";
}
$group = "GROUP BY `email`";
} elseif ($item == 'domain') {
$account_for_domains = $this->__getDomains();
$query = "SELECT `domain` AS `item`,MIN(`date`) as `oldest`,MAX(`date`) as `newest`,sum(`sent`) as `sent`,sum(`recd`) as `recd`,SUM(`sentsize`) as `sentsize`,AVG(`sentsize`) as `sentavg`,SUM(`recdsize`) as `recdsize`,AVG(`recdsize`) as `recdavg` FROM " . TABLE_STAT_COUNTER;
$where = "WHERE ( `domain` IN ('".implode("','",$account_for_domains)."') )";
if($search) {
$where .= " AND `domain` like '%".$search."%'";
}
$group = "GROUP BY `domain`";
} else {
return false;
}
if($order == 0) { $order = "ASC"; }
else { $order = "DESC"; }
$_order = "ORDER BY `$sort` $order";
$from = (int)$page * (int)$pagelen;
if($pagelen > 0) { $limit = " LIMIT " . (int)$from . ", " . (int)$pagelen; }
$query = $this->db->query($query.' '.$where.' '.$group.' '.$_order.' '.$limit.';');
if($query->num_rows >= 1) {
return $query->rows;
} else {
// no results found
return false;
}
}
public function count_accounting($item='email', $search='') {
$account_for_emails = $this->__getEmails();
$account_for_domains = $this->__getDomains();
$search = preg_replace("/\s{1,}/", "", $search);
if($search) {
$search_cond .= " AND ( `email` LIKE '%".$search."%' OR `domain` LIKE '%".$search."%' )";
}
$query = "SELECT `email` AS `item`, MIN(`date`) AS `oldest`, MAX(`date`) AS `newest`, SUM(`sent`) AS `sent`, SUM(`recd`) AS `recd`, SUM(`sentsize`) AS `sentsize`, SUM(`recdsize`) AS `recdsize` FROM " . TABLE_STAT_COUNTER;
if($item == 'email') {
$where = "WHERE `email` IN ('".implode("','",$account_for_emails)."') OR `domain` IN ('".implode("','",$account_for_domains)."')";
if($search) {
$where .= " AND ( `email` LIKE '%".$search."%' OR `domain` LIKE '%".$search."%' )";
}
$group = "GROUP BY `email`";
} elseif ($item == 'domain') {
$where = "WHERE `domain` IN ('".implode("','",$account_for_domains)."')";
if($search) {
$where .= " AND `domain` LIKE '%".$search."%'";
}
$group = "GROUP BY `domain`";
} else {
return false;
}
if($order == 0) { $order = "ASC"; }
else { $order = "DESC"; }
$query = $this->db->query($query.' '.$where.' '.$group);
$_order = "ORDER BY `$sort` $order";
$from = (int)$page * (int)$pagelen;
if($pagelen > 0) { $limit = " LIMIT " . (int)$from . ", " . (int)$pagelen; }
$query = $this->db->query($query.' '.$where.' '.$group.' '.$_order.' '.$limit.';');
if($query->num_rows >= 1)
{
return $query->rows;
} else {
// no results found
return false;
}
}
return $query->num_rows;
}
public function count_accounting($item = 'email',$search='') {
private function __getEmails() {
$emails = array();
$session = Registry::get('session');
$account_for_emails = $this->__getEmails();
$account_for_domains = $this->__getDomains();
array_push($emails, $session->get("email"));
$__emails = $session->get("emails");
$search = preg_replace("/\s{1,}/", "", $search);
foreach ($__emails as $e) {
array_push($emails, $e);
}
if($search){
$search_cond .= " AND ( `email` like '%".$search."%' OR `domain` like '%".$search."%' )";
}
$query = "SELECT `email` AS `item`,MIN(`date`) as `oldest`,MAX(`date`) as `newest`,sum(`sent`) as `sent`,sum(`recd`) as `recd`,sum(`sentsize`) as `sentsize`,sum(`recdsize`) as `recdsize` FROM " . TABLE_STAT_COUNTER;
if ($item == 'email') {
$where = "WHERE `email` IN ('".implode("','",$account_for_emails)."') OR `domain` IN ('".implode("','",$account_for_domains)."')";
if($search){
$where .= " AND ( `email` like '%".$search."%' OR `domain` like '%".$search."%' )";
}
$group = "GROUP BY `email`";
} elseif ($item == 'domain') {
$where = "WHERE `domain` IN ('".implode("','",$account_for_domains)."')";
if($search){
$where .= " AND `domain` like '%".$search."%'";
}
$group = "GROUP BY `domain`";
} else {
return false;
}
$query = $this->db->query($query.' '.$where.' '.$group.';');
return $query->num_rows;
}
private function __getEmails() {
$return = array();
$session = Registry::get('session');
array_push($return, $session->get("email"));
$emails = $session->get("emails");
foreach ($emails as $e) {
array_push($return,$e);
}
return $return;
}
private function __getDomains() {
$return = array();
$session = Registry::get('session');
if(Registry::get('admin_user') >= 1) {
$return = $this->__getAcceptedDomains();
}elseif(Registry::get('auditor_user') == 1) {
array_push($return, $session->get("domain"));
$auditdomains = $session->get("auditdomains");
foreach ($auditdomains as $d) {
array_push($return,$d);
}
}
return $return;
}
return $emails;
}
private function __getAcceptedDomains() {
// todo: move to domains model?
$return = array();
private function __getDomains() {
$domains = array();
$session = Registry::get('session');
$query = $this->db->query("SELECT domain, mapped FROM " . TABLE_DOMAIN . " ORDER BY domain ASC");
if(Registry::get('admin_user') >= 1) {
$domains = $this->model_domain_domain->get_mapped_domains();
}
elseif(Registry::get('auditor_user') == 1) {
array_push($domains, $session->get("domain"));
$auditdomains = $session->get("auditdomains");
foreach($query->rows as $domain)
{
if ($domain['domain'] == $domain['mapped']) {
array_push($return,$domain['domain']);
} else {
array_push($return,$domain['mapped']);
}
}
foreach ($auditdomains as $d) {
array_push($domains, $d);
}
}
return $return;
}
return $domains;
}
private function __decodeDate( $input ) {
if ( !is_numeric($input) )
{
// if we are passed words (yesterday, today), convert to unix timestamps representing the start of the day
$input = strtotime($input);
$return = $input - ($input%86400);
private function __decodeDate($input) {
$timestamp = 0;
} elseif ( is_numeric($input) )
{
// if we are passed unix timestamps, ensure they represent the start of the day
$return = $input - ($input%86400);
}
if(!is_numeric($input)) {
// if we got anything, but a timestamp, eg. words (yesterday, today)
// then convert to unix timestamp representing the start of the day
$input = strtotime($input);
$timestamp = $input - ($input % 86400);
}
else {
// round the timestamp to the start of the day
$timestamp = $input - ($input % 86400);
}
return $return;
return $timestamp;
}
}
}
?>