From ae532766227cb2dcc654b3f50543f67b910fdda9 Mon Sep 17 00:00:00 2001 From: SJ Date: Wed, 22 Apr 2015 12:26:04 +0200 Subject: [PATCH] added rfc3161 timestamp support --- util/Makefile.in | 1 + util/db-mysql.sql | 11 ++ util/db-upgrade-1.1.0-vs-1.2.0.sql | 11 ++ util/sign.php | 167 +++++++++++++++++ webui/config.php | 7 + webui/model/search/message.php | 61 ++++++- webui/system/helper/TrustedTimestamps.php | 209 ++++++++++++++++++++++ 7 files changed, 463 insertions(+), 4 deletions(-) create mode 100644 util/sign.php create mode 100644 webui/system/helper/TrustedTimestamps.php diff --git a/util/Makefile.in b/util/Makefile.in index 31a73b95..61f24012 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -33,6 +33,7 @@ install: $(INSTALL) -m 0755 $(srcdir)/daily-report.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/gmail-imap-import.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/generate_stats.php $(DESTDIR)$(libexecdir)/piler + $(INSTALL) -m 0755 $(srcdir)/sign.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/indexer.delta.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/indexer.main.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/import.sh $(DESTDIR)$(libexecdir)/piler diff --git a/util/db-mysql.sql b/util/db-mysql.sql index 32cdb529..2b578300 100644 --- a/util/db-mysql.sql +++ b/util/db-mysql.sql @@ -430,3 +430,14 @@ create table if not exists `legal_hold` ( ) Engine=InnoDB; +create table if not exists `timestamp` ( + `id` bigint unsigned not null auto_increment, + `start_id` bigint default 0, + `stop_id` bigint default 0, + `hash_value` char(40), + `count` int default 0, + `response_time` bigint default 0, + `response_string` blob not null, + primary key (`id`) +) Engine=InnoDB; + diff --git a/util/db-upgrade-1.1.0-vs-1.2.0.sql b/util/db-upgrade-1.1.0-vs-1.2.0.sql index 45c8e4ce..6b731e84 100644 --- a/util/db-upgrade-1.1.0-vs-1.2.0.sql +++ b/util/db-upgrade-1.1.0-vs-1.2.0.sql @@ -17,4 +17,15 @@ create table if not exists `legal_hold` ( email varchar(128) unique not null ) Engine=InnoDB; +create table if not exists `timestamp` ( + `id` bigint unsigned not null auto_increment, + `start_id` bigint default 0, + `stop_id` bigint default 0, + `hash_value` char(40), + `count` int default 0, + `response_time` bigint default 0, + `response_string` blob not null, + primary key (`id`) +) Engine=InnoDB; + diff --git a/util/sign.php b/util/sign.php new file mode 100644 index 00000000..c921a2a7 --- /dev/null +++ b/util/sign.php @@ -0,0 +1,167 @@ +query("SELECT start_id, stop_id FROM " . TABLE_TIMESTAMP . " WHERE id < 100000000000 ORDER BY id DESC LIMIT 1"); + + if($query->num_rows == 0) { + return 0; + } + + return $query->row['stop_id']; +} + + +function get_hash_values() { + $s = ''; + $count = 0; + + $db = Registry::get('db'); + + $last_id = get_last_entry_from_timestamp_table(); + + if($last_id == 0) { + $start_id = TSA_START_ID; + if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; } + else { $stop_id = 1000000000; } + } + else { + $start_id = $last_id + 1; + if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; } + else { $stop_id = 1000000000; } + } + + $query = $db->query("SELECT id, digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($start_id, $stop_id)); + + foreach($query->rows as $q) { + $count++; + $s .= $q['digest']; + } + + if(MODE == 'time') { $stop_id = $start_id + $count - 1; } + + return array( + 'start_id' => $start_id, + 'stop_id' => $stop_id, + 'count' => $count, + 'hash_value' => sha1($s) + ); + +} + + +function store_results($data = array()) { + $db = Registry::get('db'); + + $query = $db->query("INSERT INTO " . TABLE_TIMESTAMP . " (start_id, stop_id, hash_value, `count`, response_time, response_string) VALUES(?,?,?,?,?,?)", array($data['start_id'], $data['stop_id'], $data['hash_value'], $data['count'], $data['response_time'], $data['response_string'])); + + $rc = $db->countAffected(); + + return $rc; +} + + +function display_help() { + $phpself = basename(__FILE__); + echo("\nUsage: $phpself --webui [PATH] [OPTIONS...]\n\n"); + echo("\t--webui=\"[REQUIRED: path to the Piler WebUI Directory]\"\n\n"); + echo("options:\n"); + echo("\t--mode time|unit (default: unit)\n"); + echo("\t-v Provide a verbose output\n"); + echo("\t-h Prints this help screen and exits\n"); +} + + +?> diff --git a/webui/config.php b/webui/config.php index a9cc60a7..4267cb7d 100644 --- a/webui/config.php +++ b/webui/config.php @@ -187,6 +187,12 @@ $config['DECRYPT_BINARY'] = '/usr/local/bin/pilerget'; $config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/local/bin/pileraget'; $config['DECRYPT_BUFFER_LENGTH'] = 65536; +$config['OPENSSL_BINARY'] = '/usr/bin/openssl'; +$config['TSA_URL'] = ''; +$config['TSA_PUBLIC_KEY_FILE'] = ''; +$config['TSA_START_ID'] = 1; +$config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000; + $config['DB_DRIVER'] = 'mysql'; $config['DB_PREFIX'] = ''; $config['DB_HOSTNAME'] = 'localhost'; @@ -357,6 +363,7 @@ define('TABLE_GOOGLE', 'google'); define('TABLE_GOOGLE_IMAP', 'google_imap'); define('TABLE_AUTOSEARCH', 'autosearch'); define('TABLE_LEGAL_HOLD', 'legal_hold'); +define('TABLE_TIMESTAMP', 'timestamp'); define('VIEW_MESSAGES', 'v_messages'); define('EOL', "\n"); diff --git a/webui/model/search/message.php b/webui/model/search/message.php index 13712ec6..460b83f9 100644 --- a/webui/model/search/message.php +++ b/webui/model/search/message.php @@ -8,10 +8,10 @@ class ModelSearchMessage extends Model { ); - public function verify_message($id = '', $data = '') { - if($id == '') { return 0; } + public function verify_message($piler_id = '', $data = '') { + if($piler_id == '') { return 0; } - $q = $this->db->query("SELECT `size`, `hlen`, `digest`, `bodydigest`,`attachments` FROM " . TABLE_META . " WHERE piler_id=?", array($id)); + $q = $this->db->query("SELECT `size`, `hlen`, `digest`, `bodydigest`,`attachments` FROM " . TABLE_META . " WHERE piler_id=?", array($piler_id)); $digest = $q->row['digest']; $bodydigest = $q->row['bodydigest']; @@ -22,7 +22,14 @@ class ModelSearchMessage extends Model { $_digest = openssl_digest($data, "SHA256"); $_bodydigest = openssl_digest(substr($data, $hlen), "SHA256"); - if($_digest == $digest && $_bodydigest == $bodydigest) { return 1; } + if($_digest == $digest && $_bodydigest == $bodydigest) { + + if(TSA_PUBLIC_KEY_FILE) { + $id = $this->get_id_by_piler_id($piler_id); + if($this->check_rfc3161_timestamp_for_id($id) == 1) { return 1; } + } + else { return 1; } + } return 0; } @@ -762,6 +769,52 @@ class ModelSearchMessage extends Model { } + public function check_rfc3161_timestamp_for_id($id = 0) { + $s = ''; + $computed_hash = ''; + + /* + * determine which entry in the timestamp table holds the aggregated hash value, + * then compute the aggregated hash value for the digests between start_id and stop_id. + * If the hashes are the same, then verify by the public key as well + */ + + $query = $this->db->query("SELECT `start_id`, `stop_id`, `hash_value`, `response_time`, `response_string` FROM " . TABLE_TIMESTAMP . " WHERE start_id <= ? AND stop_id >= ?", array($id, $id)); + + if(isset($query->row['start_id']) && isset($query->row['stop_id'])) { + + if(MEMCACHED_ENABLED) { + $cache_key = "rfc3161_" . $query->row['start_id'] . "+" . $query->row['stop_id']; + $memcache = Registry::get('memcache'); + $computed_hash = $memcache->get($cache_key); + } + + if($computed_hash == '') { + + $query2 = $this->db->query("SELECT digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($query->row['start_id'], $query->row['stop_id'])); + + foreach($query2->rows as $q) { + $s .= $q['digest']; + } + + $computed_hash = sha1($s); + + if(MEMCACHED_ENABLED) { + $memcache->add($cache_key, $computed_hash, 0, MEMCACHED_TTL); + } + } + + if($query->row['hash_value'] == $computed_hash) { + $validate = TrustedTimestamps::validate($query->row['hash_value'], $query->row['response_string'], $query->row['response_time'], TSA_PUBLIC_KEY_FILE); + if($validate == true) { return 1; } + } + + } + + return 0; + } + + public function get_attachment_list($piler_id = 0) { $data = array(); diff --git a/webui/system/helper/TrustedTimestamps.php b/webui/system/helper/TrustedTimestamps.php new file mode 100644 index 00000000..6ad00d65 --- /dev/null +++ b/webui/system/helper/TrustedTimestamps.php @@ -0,0 +1,209 @@ += 0.99 + * This is currently (2011-03-02) not the case in Debian + * (see http://stackoverflow.com/questions/5043393/openssl-ts-command-not-working-trusted-timestamps) + * -> Possibility: Debian Experimentals -> http://wiki.debian.org/DebianExperimental + * + * For OpenSSL on Windows, see + * http://www.slproweb.com/products/Win32OpenSSL.html + * http://www.switch.ch/aai/support/howto/openssl-windows.html + * + * @version 0.3 + * @author David Müller + * @package trustedtimestamps +*/ + +class TrustedTimestamps +{ + /** + * Creates a Timestamp Requestfile from a hash + * + * @param string $hash: The hashed data (sha1) + * @return string: path of the created timestamp-requestfile + */ + public static function createRequestfile ($hash) + { + if (strlen($hash) !== 40) + throw new Exception("Invalid Hash."); + + $outfilepath = self::createTempFile(); + $cmd = OPENSSL_BINARY . " ts -query -digest ".escapeshellarg($hash)." -cert -out ".escapeshellarg($outfilepath); + + $retarray = array(); + exec($cmd." 2>&1", $retarray, $retcode); + + if ($retcode !== 0) + throw new Exception("OpenSSL does not seem to be installed: ".implode(", ", $retarray)); + + if (count($retarray) > 0 && stripos($retarray[0], "openssl:Error") !== false) + throw new Exception("There was an error with OpenSSL. Is version >= 0.99 installed?: ".implode(", ", $retarray)); + + return $outfilepath; + } + + /** + * Signs a timestamp requestfile at a TSA using CURL + * + * @param string $requestfile_path: The path to the Timestamp Requestfile as created by createRequestfile + * @param string $tsa_url: URL of a TSA such as http://zeitstempel.dfn.de + * @return array of response_string with the unix-timetamp of the timestamp response and the base64-encoded response_string + */ + public static function signRequestfile ($requestfile_path, $tsa_url) + { + if (!file_exists($requestfile_path)) + throw new Exception("The Requestfile was not found"); + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $tsa_url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_TIMEOUT, 10); + curl_setopt($ch, CURLOPT_POST, 1); + curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1); + curl_setopt($ch, CURLOPT_POSTFIELDS, file_get_contents($requestfile_path)); + curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/timestamp-query')); + curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)"); + $binary_response_string = curl_exec($ch); + $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if ($status != 200 || !strlen($binary_response_string)) + throw new Exception("The request failed"); + + $base64_response_string = base64_encode($binary_response_string); + + $response_time = self::getTimestampFromAnswer ($base64_response_string); + + return array("response_string" => $base64_response_string, + "response_time" => $response_time); + } + + /** + * Extracts the unix timestamp from the base64-encoded response string as returned by signRequestfile + * + * @param string $base64_response_string: Response string as returned by signRequestfile + * @return int: unix timestamp + */ + public static function getTimestampFromAnswer ($base64_response_string) + { + $binary_response_string = base64_decode($base64_response_string); + + $responsefile = self::createTempFile($binary_response_string); + + $cmd = OPENSSL_BINARY . " ts -reply -in ".escapeshellarg($responsefile)." -text"; + + $retarray = array(); + exec($cmd." 2>&1", $retarray, $retcode); + + if ($retcode !== 0) + throw new Exception("The reply failed: ".implode(", ", $retarray)); + + $matches = array(); + $response_time = 0; + + /* + * Format of answer: + * + * Foobar: some stuff + * Time stamp: 21.08.2010 blabla GMT + * Somestuff: Yayayayaya + */ + foreach ($retarray as $retline) + { + if (preg_match("~^Time\sstamp\:\s(.*)~", $retline, $matches)) + { + $response_time = strtotime($matches[1]); + break; + } + } + + if (!$response_time) + throw new Exception("The Timestamp was not found"); + + return $response_time; + } + + /** + * + * @param string $hash: sha1 hash of the data which should be checked + * @param string $base64_response_string: The response string as returned by signRequestfile + * @param int $response_time: The response time, which should be checked + * @param string $tsa_cert_file: The path to the TSAs certificate chain (e.g. https://pki.pca.dfn.de/global-services-ca/pub/cacert/chain.txt) + * @return + */ + public static function validate ($hash, $base64_response_string, $response_time, $tsa_cert_file) + { + if (strlen($hash) !== 40) + throw new Exception("Invalid Hash"); + + $binary_response_string = base64_decode($base64_response_string); + + if (!strlen($binary_response_string)) + throw new Exception("There was no response-string"); + + if (!intval($response_time)) + throw new Exception("There is no valid response-time given"); + + if (!file_exists($tsa_cert_file)) + throw new Exception("The TSA-Certificate could not be found"); + + $responsefile = self::createTempFile($binary_response_string); + + $cmd = OPENSSL_BINARY . " ts -verify -digest ".escapeshellarg($hash)." -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file); + + $retarray = array(); + exec($cmd." 2>&1", $retarray, $retcode); + + /* + * just 2 "normal" cases: + * 1) Everything okay -> retcode 0 + retarray[0] == "Verification: OK" + * 2) Hash is wrong -> retcode 1 + strpos(retarray[somewhere], "message imprint mismatch") !== false + * + * every other case (Certificate not found / invalid / openssl is not installed / ts command not known) + * are being handled the same way -> retcode 1 + any retarray NOT containing "message imprint mismatch" + */ + + if ($retcode === 0 && strtolower(trim($retarray[0])) == "verification: ok") + { + if (self::getTimestampFromAnswer ($base64_response_string) != $response_time) + throw new Exception("The responsetime of the request was changed"); + + return true; + } + + foreach ($retarray as $retline) + { + if (stripos($retline, "message imprint mismatch") !== false) + return false; + } + + throw new Exception("Systemcommand failed: ".implode(", ", $retarray)); + } + + /** + * Create a tempfile in the systems temp path + * + * @param string $str: Content which should be written to the newly created tempfile + * @return string: filepath of the created tempfile + */ + public static function createTempFile ($str = "") + { + $tempfilename = tempnam(sys_get_temp_dir(), rand()); + + if (!file_exists($tempfilename)) + throw new Exception("Tempfile could not be created"); + + if (!empty($str) && !file_put_contents($tempfilename, $str)) + throw new Exception("Could not write to tempfile"); + + return $tempfilename; + } +} + +?>