added rfc3161 timestamp support

util/Makefile.in

@@ -33,6 +33,7 @@ install:
 	$(INSTALL) -m 0755 $(srcdir)/daily-report.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/gmail-imap-import.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/generate_stats.php $(DESTDIR)$(libexecdir)/piler
+	$(INSTALL) -m 0755 $(srcdir)/sign.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/indexer.delta.sh $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/indexer.main.sh $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/import.sh $(DESTDIR)$(libexecdir)/piler

util/db-mysql.sql

@@ -430,3 +430,14 @@ create table if not exists `legal_hold` (
 ) Engine=InnoDB;
 
 
+create table if not exists `timestamp` (
+  `id` bigint unsigned not null auto_increment,
+  `start_id` bigint default 0,
+  `stop_id` bigint default 0,
+  `hash_value` char(40),
+  `count` int default 0,
+  `response_time` bigint default 0,
+  `response_string` blob not null,
+  primary key (`id`)
+) Engine=InnoDB;
+

util/db-upgrade-1.1.0-vs-1.2.0.sql

@@ -17,4 +17,15 @@ create table if not exists `legal_hold` (
    email varchar(128) unique not null
 ) Engine=InnoDB;
 
+create table if not exists `timestamp` (
+  `id` bigint unsigned not null auto_increment,
+  `start_id` bigint default 0,
+  `stop_id` bigint default 0,
+  `hash_value` char(40),
+  `count` int default 0,
+  `response_time` bigint default 0,
+  `response_string` blob not null,
+  primary key (`id`)
+) Engine=InnoDB;
+
 

util/sign.php

@@ -0,0 +1,167 @@
+<?php
+
+ini_set("session.save_path", "/tmp");
+
+$webuidir = "";
+$verbose = 0;
+$mode = "unit";
+
+$opts = 'h::v';
+$lopts = array(
+    'webui:',
+    'mode:',
+    'verbose'
+    );
+    
+if ( $options = getopt( $opts, $lopts ) )
+{
+    if ( isset($options['webui']) ) 
+    {
+        $webuidir = $options['webui'];
+    } else
+    {
+        echo("\nError: must provide path to WebUI directory\n\n");
+    
+        display_help();
+        exit;
+    }
+   
+    if ( isset($options['mode']) )
+    {
+       if($options['mode'] == 'time') { $mode = $options['mode']; }
+    }
+ 
+    if ( isset($options['h']) ) 
+    {
+        display_help();
+        exit;
+    }
+    if ( isset($options['verbose']) )
+    {
+        $verbose = 1;
+    }
+} else {
+    display_help();
+    exit;   
+}
+
+require_once($webuidir . "/config.php");
+require_once(DIR_SYSTEM . "/helper/TrustedTimestamps.php");
+
+if(ENABLE_SYSLOG == 1) { openlog("piler-timestamp", LOG_PID, LOG_MAIL); }
+
+require(DIR_SYSTEM . "/startup.php");
+
+$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
+Registry::set('DB_DATABASE', DB_DATABASE);
+
+Registry::set('db', $db);
+
+Registry::set('DB_DRIVER', DB_DRIVER);
+
+define('MODE', $mode);
+
+
+$data = get_hash_values();
+
+//print_r($data);
+
+// check if we have enough messages
+
+if(MODE == 'unit' && $data['count'] < TSA_STAMP_REQUEST_UNIT_SIZE) {
+   syslog(LOG_INFO, "not enough messages to sign yet (" . $data['count'] . ")");
+   exit;
+}
+
+if(MODE == 'time' && $data['count'] < 1) {
+   syslog(LOG_INFO, "not enough messages to sign yet (" . $data['count'] . ")");
+   exit;
+}
+
+$requestfile_path = TrustedTimestamps::createRequestfile($data['hash_value']);
+
+$response = TrustedTimestamps::signRequestfile($requestfile_path, TSA_URL);
+
+$data['response_string'] = $response['response_string'];
+$data['response_time'] = $response['response_time'];
+
+$rc = store_results($data);
+if($rc == 1) { syslog(LOG_INFO, "signed " . $data['count'] . " messages (" . $data['start_id'] . "-" . $data['stop_id'] . ") on @" . $data['response_time']); }
+else { syslog(LOG_INFO, "ERROR: failed to sign " . $data['count'] . " messages (" . $data['start_id'] . "-" . $data['stop_id'] . ") on @" . $data['response_time']); }
+
+
+
+function get_last_entry_from_timestamp_table() {
+   $db = Registry::get('db');
+
+   $query = $db->query("SELECT start_id, stop_id FROM " . TABLE_TIMESTAMP . " WHERE id < 100000000000 ORDER BY id DESC LIMIT 1");
+
+   if($query->num_rows == 0) {
+      return 0;
+   }
+
+   return $query->row['stop_id'];
+}
+
+
+function get_hash_values() {
+   $s = '';
+   $count = 0;
+
+   $db = Registry::get('db');
+
+   $last_id = get_last_entry_from_timestamp_table();
+
+   if($last_id == 0) {
+      $start_id = TSA_START_ID;
+      if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; }
+      else { $stop_id = 1000000000; }
+   }
+   else {
+      $start_id = $last_id + 1;
+      if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; }
+      else { $stop_id = 1000000000; }
+   }
+
+   $query = $db->query("SELECT id, digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($start_id, $stop_id));
+
+   foreach($query->rows as $q) {
+      $count++;
+      $s .= $q['digest'];
+   }
+
+   if(MODE == 'time') { $stop_id = $start_id + $count - 1; }
+
+   return array(
+                'start_id' => $start_id,
+                'stop_id' => $stop_id,
+                'count' => $count,
+                'hash_value' => sha1($s)
+          );
+
+}
+
+
+function store_results($data = array()) {
+   $db = Registry::get('db');
+
+   $query = $db->query("INSERT INTO " . TABLE_TIMESTAMP . " (start_id, stop_id, hash_value, `count`, response_time, response_string) VALUES(?,?,?,?,?,?)", array($data['start_id'], $data['stop_id'], $data['hash_value'], $data['count'], $data['response_time'], $data['response_string']));
+
+   $rc = $db->countAffected();
+
+   return $rc;
+}
+
+
+function display_help() {
+    $phpself = basename(__FILE__);
+    echo("\nUsage: $phpself --webui [PATH] [OPTIONS...]\n\n");
+    echo("\t--webui=\"[REQUIRED: path to the Piler WebUI Directory]\"\n\n");
+    echo("options:\n");
+    echo("\t--mode time|unit (default: unit)\n");
+    echo("\t-v Provide a verbose output\n");
+    echo("\t-h Prints this help screen and exits\n");
+}
+
+
+?>

webui/config.php

@@ -187,6 +187,12 @@ $config['DECRYPT_BINARY'] = '/usr/local/bin/pilerget';
 $config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/local/bin/pileraget';
 $config['DECRYPT_BUFFER_LENGTH'] = 65536;
 
+$config['OPENSSL_BINARY'] = '/usr/bin/openssl';
+$config['TSA_URL'] = '';
+$config['TSA_PUBLIC_KEY_FILE'] = '';
+$config['TSA_START_ID'] = 1;
+$config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
+
 $config['DB_DRIVER'] = 'mysql';
 $config['DB_PREFIX'] = '';
 $config['DB_HOSTNAME'] = 'localhost';
@@ -357,6 +363,7 @@ define('TABLE_GOOGLE', 'google');
 define('TABLE_GOOGLE_IMAP', 'google_imap');
 define('TABLE_AUTOSEARCH', 'autosearch');
 define('TABLE_LEGAL_HOLD', 'legal_hold');
+define('TABLE_TIMESTAMP', 'timestamp');
 define('VIEW_MESSAGES', 'v_messages');
 
 define('EOL', "\n");

webui/model/search/message.php

@@ -8,10 +8,10 @@ class ModelSearchMessage extends Model {
                                    );
 
 
-   public function verify_message($id = '', $data = '') {
+   public function verify_message($piler_id = '', $data = '') {
-      if($id == '') { return 0; }
+      if($piler_id == '') { return 0; }
 
-      $q = $this->db->query("SELECT `size`, `hlen`, `digest`, `bodydigest`,`attachments` FROM " . TABLE_META . " WHERE piler_id=?", array($id));
+      $q = $this->db->query("SELECT `size`, `hlen`, `digest`, `bodydigest`,`attachments` FROM " . TABLE_META . " WHERE piler_id=?", array($piler_id));
 
       $digest = $q->row['digest'];
       $bodydigest = $q->row['bodydigest'];
@@ -22,7 +22,14 @@ class ModelSearchMessage extends Model {
       $_digest = openssl_digest($data, "SHA256");
       $_bodydigest = openssl_digest(substr($data, $hlen), "SHA256");
 
-      if($_digest == $digest && $_bodydigest == $bodydigest) { return 1; }
+      if($_digest == $digest && $_bodydigest == $bodydigest) {
+
+         if(TSA_PUBLIC_KEY_FILE) {
+            $id = $this->get_id_by_piler_id($piler_id);
+            if($this->check_rfc3161_timestamp_for_id($id) == 1) { return 1; }
+         }
+         else { return 1; }
+      }
 
       return 0;
    }
@@ -762,6 +769,52 @@ class ModelSearchMessage extends Model {
    }
 
 
+   public function check_rfc3161_timestamp_for_id($id = 0) {
+      $s = '';
+      $computed_hash = '';
+
+      /*
+       * determine which entry in the timestamp table holds the aggregated hash value,
+       * then compute the aggregated hash value for the digests between start_id and stop_id.
+       * If the hashes are the same, then verify by the public key as well
+       */
+
+      $query = $this->db->query("SELECT `start_id`, `stop_id`, `hash_value`, `response_time`, `response_string` FROM " . TABLE_TIMESTAMP . " WHERE start_id <= ? AND stop_id >= ?", array($id, $id));
+
+      if(isset($query->row['start_id']) && isset($query->row['stop_id'])) {
+
+         if(MEMCACHED_ENABLED) {
+            $cache_key = "rfc3161_" . $query->row['start_id'] . "+" . $query->row['stop_id'];
+            $memcache = Registry::get('memcache');
+            $computed_hash = $memcache->get($cache_key);
+         }
+
+         if($computed_hash == '') {
+
+            $query2 = $this->db->query("SELECT digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($query->row['start_id'], $query->row['stop_id']));
+
+            foreach($query2->rows as $q) {
+               $s .= $q['digest'];
+            }
+
+            $computed_hash = sha1($s);
+
+            if(MEMCACHED_ENABLED) {
+               $memcache->add($cache_key, $computed_hash, 0, MEMCACHED_TTL);
+            }
+         }
+
+         if($query->row['hash_value'] == $computed_hash) {
+            $validate = TrustedTimestamps::validate($query->row['hash_value'], $query->row['response_string'], $query->row['response_time'], TSA_PUBLIC_KEY_FILE);
+            if($validate == true) { return 1; }
+         }
+
+      }
+
+      return 0;
+   }
+
+
    public function get_attachment_list($piler_id = 0) {
       $data = array();
 

webui/system/helper/TrustedTimestamps.php

@@ -0,0 +1,209 @@
+<?php
+
+/**
+ * TrustedTimestamps.php - Creates Timestamp Requestfiles, processes the request at a Timestamp Authority (TSA) after RFC 3161
+ *
+ * bases on OpenSSL and RFC 3161: http://www.ietf.org/rfc/rfc3161.txt
+ *
+ * WARNING:
+ *  needs openssl ts, which is availible in OpenSSL versions >= 0.99
+ *  This is currently (2011-03-02) not the case in Debian
+ *  (see http://stackoverflow.com/questions/5043393/openssl-ts-command-not-working-trusted-timestamps)
+ *  -> Possibility: Debian Experimentals -> http://wiki.debian.org/DebianExperimental
+ *
+ * For OpenSSL on Windows, see
+ *  http://www.slproweb.com/products/Win32OpenSSL.html
+ *  http://www.switch.ch/aai/support/howto/openssl-windows.html
+ *
+ * @version 0.3
+ * @author David Müller
+ * @package trustedtimestamps
+*/
+
+class TrustedTimestamps
+{
+    /**
+     * Creates a Timestamp Requestfile from a hash
+     *
+     * @param string $hash: The hashed data (sha1)
+     * @return string: path of the created timestamp-requestfile
+     */
+    public static function createRequestfile ($hash)
+    {
+        if (strlen($hash) !== 40)
+            throw new Exception("Invalid Hash.");
+           
+        $outfilepath = self::createTempFile();
+        $cmd = OPENSSL_BINARY . " ts -query -digest ".escapeshellarg($hash)." -cert -out ".escapeshellarg($outfilepath);
+
+        $retarray = array();
+        exec($cmd." 2>&1", $retarray, $retcode);
+
+        if ($retcode !== 0)
+            throw new Exception("OpenSSL does not seem to be installed: ".implode(", ", $retarray));
+       
+        if (count($retarray) > 0 && stripos($retarray[0], "openssl:Error") !== false)
+            throw new Exception("There was an error with OpenSSL. Is version >= 0.99 installed?: ".implode(", ", $retarray));
+
+        return $outfilepath;
+    }
+
+    /**
+     * Signs a timestamp requestfile at a TSA using CURL
+     *
+     * @param string $requestfile_path: The path to the Timestamp Requestfile as created by createRequestfile
+     * @param string $tsa_url: URL of a TSA such as http://zeitstempel.dfn.de
+     * @return array of response_string with the unix-timetamp of the timestamp response and the base64-encoded response_string
+     */
+    public static function signRequestfile ($requestfile_path, $tsa_url)
+    {
+        if (!file_exists($requestfile_path))
+            throw new Exception("The Requestfile was not found");
+
+        $ch = curl_init();
+        curl_setopt($ch, CURLOPT_URL, $tsa_url);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+        curl_setopt($ch, CURLOPT_TIMEOUT, 10);
+        curl_setopt($ch, CURLOPT_POST, 1);
+        curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1);
+        curl_setopt($ch, CURLOPT_POSTFIELDS, file_get_contents($requestfile_path));
+        curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/timestamp-query'));
+        curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)");
+        $binary_response_string = curl_exec($ch);
+        $status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+        curl_close($ch);
+       
+        if ($status != 200 || !strlen($binary_response_string))
+            throw new Exception("The request failed");
+       
+        $base64_response_string = base64_encode($binary_response_string);
+       
+        $response_time = self::getTimestampFromAnswer ($base64_response_string);
+       
+        return array("response_string" => $base64_response_string,
+                     "response_time" => $response_time);
+    }
+
+    /**
+     * Extracts the unix timestamp from the base64-encoded response string as returned by signRequestfile
+     *
+     * @param string $base64_response_string: Response string as returned by signRequestfile
+     * @return int: unix timestamp
+     */
+    public static function getTimestampFromAnswer ($base64_response_string)
+    {
+        $binary_response_string = base64_decode($base64_response_string);
+
+        $responsefile = self::createTempFile($binary_response_string);
+
+        $cmd = OPENSSL_BINARY . " ts -reply -in ".escapeshellarg($responsefile)." -text";
+       
+        $retarray = array();
+        exec($cmd." 2>&1", $retarray, $retcode);
+       
+        if ($retcode !== 0)
+            throw new Exception("The reply failed: ".implode(", ", $retarray));
+       
+        $matches = array();
+        $response_time = 0;
+
+        /*
+         * Format of answer:
+         *
+         * Foobar: some stuff
+         * Time stamp: 21.08.2010 blabla GMT
+         * Somestuff: Yayayayaya
+         */
+        foreach ($retarray as $retline)
+        {
+            if (preg_match("~^Time\sstamp\:\s(.*)~", $retline, $matches))
+            {
+                $response_time = strtotime($matches[1]);
+                break;     
+            }
+        }
+
+        if (!$response_time)
+            throw new Exception("The Timestamp was not found");
+           
+        return $response_time;
+    }
+
+    /**
+     *
+     * @param string $hash: sha1 hash of the data which should be checked
+     * @param string $base64_response_string: The response string as returned by signRequestfile
+     * @param int $response_time: The response time, which should be checked
+     * @param string $tsa_cert_file: The path to the TSAs certificate chain (e.g. https://pki.pca.dfn.de/global-services-ca/pub/cacert/chain.txt)
+     * @return <type>
+     */
+    public static function validate ($hash, $base64_response_string, $response_time, $tsa_cert_file)
+    {
+        if (strlen($hash) !== 40)
+            throw new Exception("Invalid Hash");
+       
+        $binary_response_string = base64_decode($base64_response_string);
+       
+        if (!strlen($binary_response_string))
+            throw new Exception("There was no response-string");   
+           
+        if (!intval($response_time))
+            throw new Exception("There is no valid response-time given");
+       
+        if (!file_exists($tsa_cert_file))
+            throw new Exception("The TSA-Certificate could not be found");
+       
+        $responsefile = self::createTempFile($binary_response_string);
+
+        $cmd = OPENSSL_BINARY . " ts -verify -digest ".escapeshellarg($hash)." -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file);
+       
+        $retarray = array();
+        exec($cmd." 2>&1", $retarray, $retcode);
+       
+        /*
+         * just 2 "normal" cases:
+         *  1) Everything okay -> retcode 0 + retarray[0] == "Verification: OK"
+         *  2) Hash is wrong -> retcode 1 + strpos(retarray[somewhere], "message imprint mismatch") !== false
+         *
+         * every other case (Certificate not found / invalid / openssl is not installed / ts command not known)
+         * are being handled the same way -> retcode 1 + any retarray NOT containing "message imprint mismatch"
+         */
+       
+        if ($retcode === 0 && strtolower(trim($retarray[0])) == "verification: ok")
+        {
+            if (self::getTimestampFromAnswer ($base64_response_string) != $response_time)
+                throw new Exception("The responsetime of the request was changed");
+           
+            return true;
+        }
+
+        foreach ($retarray as $retline)
+        {
+            if (stripos($retline, "message imprint mismatch") !== false)
+                return false;
+        }
+
+        throw new Exception("Systemcommand failed: ".implode(", ", $retarray));
+    }
+
+    /**
+     * Create a tempfile in the systems temp path
+     *
+     * @param string $str: Content which should be written to the newly created tempfile
+     * @return string: filepath of the created tempfile
+     */
+    public static function createTempFile ($str = "")
+    {
+        $tempfilename = tempnam(sys_get_temp_dir(), rand());
+
+        if (!file_exists($tempfilename))
+            throw new Exception("Tempfile could not be created");
+           
+        if (!empty($str) && !file_put_contents($tempfilename, $str))
+            throw new Exception("Could not write to tempfile");
+
+        return $tempfilename;
+    }
+}
+
+?>