added customer support for the piler daemon

configure

@@ -697,6 +697,7 @@ enable_clamd
 enable_memcached
 enable_starttls
 enable_tcpwrappers
+enable_multitenancy
 enable_tweak_sent_time
 with_piler_user
 '
@@ -1321,6 +1322,7 @@ Optional Features:
   --enable-memcached      build memcached support
   --enable-starttls       build starttls support
   --enable-tcpwrappers    build tcpwrappers support
+  --enable-multitenancy   build piler with multitenancy support (!!!EXPERIMENTAL!!!)
   --tweak-sent-time       tweak sent time
 
 Optional Packages:
@@ -3440,6 +3442,7 @@ have_zlib="no"
 have_starttls="no"
 have_tcpwrappers="no"
 have_tweak_sent_time="no"
+have_multitenancy="no"
 
 pdftotext="no"
 catdoc="no"
@@ -3638,6 +3641,16 @@ ac_cv_lib_wrap=ac_cv_lib_wrap_main
 fi
 
 
+# Check whether --enable-multitenancy was given.
+if test "${enable_multitenancy+set}" = set; then :
+  enableval=$enable_multitenancy; have_multitenancy=$enableval
+else
+  have_multitenancy="no"
+fi
+
+
+
+
 # Check whether --enable-tweak-sent-time was given.
 if test "${enable_tweak_sent_time+set}" = set; then :
   enableval=$enable_tweak_sent_time; have_tweak_sent_time=$enableval
@@ -4545,6 +4558,13 @@ if test "$have_icc_guide" = "yes" && test "$have_mysql" = "yes"; then
    mysql_libs="$mysql_libs -lguide"
 fi
 
+if test "$have_multitenancy" = "yes"; then
+   echo "multitenancy support: yes"
+   defs="$defs -DHAVE_MULTITENANCY"
+   objs="$objs multitenancy.o"
+else
+   echo "multitenancy support: no"
+fi
 
 if test z`which pdftotext 2>/dev/null` != "z"; then
    pdftotext=`which pdftotext`
@@ -4555,7 +4575,6 @@ _ACEOF
 
 fi
 
-
 if test z`which catdoc 2>/dev/null` != "z"; then
    catdoc=`which catdoc`
 

configure.in

@@ -43,6 +43,7 @@ have_zlib="no"
 have_starttls="no"
 have_tcpwrappers="no"
 have_tweak_sent_time="no"
+have_multitenancy="no"
 
 pdftotext="no"
 catdoc="no"
@@ -125,6 +126,11 @@ if test "$want_tcpwrappers" = "yes"; then
 fi
 
 
+AC_ARG_ENABLE(multitenancy,
+	[  --enable-multitenancy   build piler with multitenancy support (!!!EXPERIMENTAL!!!)], have_multitenancy=$enableval, have_multitenancy="no")
+
+
+
 AC_ARG_ENABLE(tweak-sent-time,
 	[  --tweak-sent-time       tweak sent time], have_tweak_sent_time=$enableval, have_tweak_sent_time="no")
 
@@ -344,13 +350,19 @@ if test "$have_icc_guide" = "yes" && test "$have_mysql" = "yes"; then
    mysql_libs="$mysql_libs -lguide"
 fi
 
+if test "$have_multitenancy" = "yes"; then
+   echo "multitenancy support: yes"
+   defs="$defs -DHAVE_MULTITENANCY"
+   objs="$objs multitenancy.o"
+else
+   echo "multitenancy support: no"
+fi
 
 if test z`which pdftotext 2>/dev/null` != "z"; then
    pdftotext=`which pdftotext`
    AC_DEFINE_UNQUOTED(HAVE_PDFTOTEXT, "$pdftotext", [path to pdftotext])
 fi
 
-
 if test z`which catdoc 2>/dev/null` != "z"; then
    catdoc=`which catdoc`
    AC_DEFINE_UNQUOTED(HAVE_CATDOC, "$catdoc", [path to catdoc])

src/config.h

@@ -86,13 +86,21 @@
 #define SQL_COUNTER_TABLE "counter"
 #define SQL_OPTION_TABLE "option"
 #define SQL_DOMAIN_TABLE "domain"
+#define SQL_CUSTOMER_TABLE "customer"
 #define SQL_MESSAGES_VIEW "v_messages"
 #define SQL_ATTACHMENTS_VIEW "v_attachment"
 
 #define SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID  "SELECT id FROM " SQL_METADATA_TABLE " WHERE message_id=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE     "INSERT INTO " SQL_RECIPIENT_TABLE " (`id`,`to`,`todomain`) VALUES(?,?,?)"
-#define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE   "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+
-#define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE     "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+#ifdef HAVE_MULTITENANCY
+   #define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE   "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+   #define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE     "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+#else
+   #define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE   "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`, `customer`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+   #define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE     "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`,`customer`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+#endif
+
 #define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE     "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)"
 #define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE   "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=?"
 #define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER     "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?"

src/defs.h

@@ -181,6 +181,7 @@ struct session_data {
    char attachments[SMALLBUFSIZE];
    char internal_sender, internal_recipient, external_recipient;
    int direction;
+   short int customer_id;
    int tls;
    int spam_message;
    int fd, hdr_len, tot_len, num_of_rcpt_to, rav;
@@ -271,6 +272,11 @@ struct __data {
 
    SSL_CTX *ctx;
    SSL *ssl;
+
+#ifdef HAVE_MULTITENANCY
+   struct list *customers;
+#endif
+
 };
 
 

src/message.c

@@ -21,8 +21,8 @@ int store_index_data(struct session_data *sdata, struct _state *state, struct __
    int rc=ERR;
    char *subj;
 
-   MYSQL_BIND bind[14];
+   MYSQL_BIND bind[15];
-   unsigned long len[14];
+   unsigned long len[15];
 
    subj = state->b_subject;
    if(*subj == ' ') subj++;
@@ -110,6 +110,14 @@ int store_index_data(struct session_data *sdata, struct _state *state, struct __
    len[13] = strlen(sdata->attachments); bind[13].length = &len[13];
 
 
+#ifdef HAVE_MULTITENANCY
+   bind[14].buffer_type = MYSQL_TYPE_SHORT;
+   bind[14].buffer = (char *)&sdata->customer_id;
+   bind[14].is_null = 0;
+   bind[14].length = 0;
+#endif
+
+
    if(mysql_stmt_bind_param(data->stmt_insert_into_sphinx_table, bind)){
       syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(data->stmt_insert_into_sphinx_table));
       goto CLOSE;
@@ -270,8 +278,8 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __d
    int rc, ret=ERR;
    char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
 
-   MYSQL_BIND bind[17];
+   MYSQL_BIND bind[18];
-   unsigned long len[17];
+   unsigned long len[18];
 
    my_ulonglong id=0;
 
@@ -396,6 +404,12 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __d
    bind[16].is_null = 0;
    len[16] = strlen(vcode); bind[16].length = &len[16];
 
+#ifdef HAVE_MULTITENANCY
+   bind[17].buffer_type = MYSQL_TYPE_SHORT;
+   bind[17].buffer = (char *)&sdata->customer_id;
+   bind[17].is_null = 0;
+   bind[17].length = 0;
+#endif
 
 
    if(mysql_stmt_bind_param(data->stmt_insert_into_meta_table, bind)){

src/misc.c

@@ -476,6 +476,8 @@ void init_session_data(struct session_data *sdata, struct __config *cfg){
 
    sdata->spam_message = 0;
 
+   sdata->customer_id = 0;
+
    sdata->__acquire = sdata->__parsed = sdata->__av = sdata->__store = sdata->__compress = sdata->__encrypt = 0;
 
 

src/multitenancy.c

@@ -0,0 +1,69 @@
+/*
+ * multitenancy.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+void load_customers(struct session_data *sdata, struct __data *data, struct __config *cfg){
+   int rc;
+   char s[SMALLBUFSIZE];
+   MYSQL_RES *res;
+   MYSQL_ROW row;
+
+   data->customers = NULL;
+
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `guid` FROM `%s`", SQL_CUSTOMER_TABLE);
+
+   if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
+      res = mysql_store_result(&(sdata->mysql));
+      if(res != NULL){
+         while((row = mysql_fetch_row(res))){
+            snprintf(s, sizeof(s)-1, "%s@%d", (char*)row[1], atoi(row[0]));
+            rc = append_list(&(data->customers), s);
+            if(cfg->verbosity >= _LOG_DEBUG && rc == 1) syslog(LOG_PRIORITY, "adding customer: id: %d, guid: '%s'", atoi(row[0]), row[1]);
+            if(rc == -1) syslog(LOG_PRIORITY, "error: failed to add customer: id: %d, guid: '%s'", atoi(row[0]), row[1]);
+         }
+
+         mysql_free_result(res);
+      }
+
+   }
+
+}
+
+
+unsigned char get_customer_id_by_rcpt_to_email(char *rcpttoemail, struct __data *data){
+   int len;
+   char *q;
+   struct list *p;
+
+   q = strchr(rcpttoemail, '@');
+   if(!q || strlen(q) < 3) return 0;
+
+   len = strlen(rcpttoemail) - strlen(q) + 1;
+   if(len < 5) return 0;
+
+   p = data->customers;
+
+   while(p != NULL){
+      if(strncasecmp(p->s, rcpttoemail, len) == 0){
+         q = strchr(p->s, '@');
+
+         if(!q || strlen(q) < 2) return 0;
+
+         return atoi(q+1);
+      }
+
+      p = p->r;
+   }
+
+   return 0;
+}
+
+

src/piler.c

@@ -231,6 +231,10 @@ void p_clean_exit(){
    free_rule(data.archiving_rules);
    free_rule(data.retention_rules);
 
+#ifdef HAVE_MULTITENANCY
+   free_list(data.customers);
+#endif
+
    syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
 
    unlink(cfg.pidfile);
@@ -304,10 +308,17 @@ void initialise_configuration(){
    free_rule(data.archiving_rules);
    free_rule(data.retention_rules);
 
+#ifdef HAVE_MULTITENANCY
+   free_list(data.customers);
+#endif
+
    data.folder = 0;
    data.recursive_folder_names = 0;
    data.archiving_rules = NULL;
    data.retention_rules = NULL;
+#ifdef HAVE_MULTITENANCY
+   data.customers = NULL;
+#endif
 
    memset(data.starttls, 0, sizeof(data.starttls));
 
@@ -327,6 +338,10 @@ void initialise_configuration(){
 
    load_mydomains(&sdata, &data, &cfg);
 
+#ifdef HAVE_MULTITENANCY
+   load_customers(&sdata, &data, &cfg);
+#endif
+
    if(cfg.server_id > 0) insert_offset(&sdata, cfg.server_id);
 
    close_database(&sdata);
@@ -374,6 +389,10 @@ int main(int argc, char **argv){
    data.recursive_folder_names = 0;
    data.archiving_rules = NULL;
    data.retention_rules = NULL;
+#ifdef HAVE_MULTITENANCY
+   data.customers = NULL;
+#endif
+
    data.ctx = NULL;
    data.ssl = NULL;
 

src/piler.h

@@ -68,6 +68,8 @@ void extract_attachment_content(struct session_data *sdata, struct _state *state
 int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg);
 
 void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg);
+void load_customers(struct session_data *sdata, struct __data *data, struct __config *cfg);
+unsigned char get_customer_id_by_rcpt_to_email(char *rcpttoemail, struct __data *data);
 
 int create_prepared_statements(struct session_data *sdata, struct __data *data);
 void close_prepared_statements(struct __data *data);

src/session.c

@@ -22,7 +22,7 @@
 int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){
    int i, ret, pos, n, inj=ERR, state, prevlen=0;
    char *p, buf[MAXBUFSIZE], puf[MAXBUFSIZE], resp[MAXBUFSIZE], prevbuf[MAXBUFSIZE], last2buf[2*MAXBUFSIZE+1];
-   char rctptoemail[SMALLBUFSIZE], virusinfo[SMALLBUFSIZE], delay[SMALLBUFSIZE];
+   char rcpttoemail[SMALLBUFSIZE], virusinfo[SMALLBUFSIZE], delay[SMALLBUFSIZE];
    char *arule = NULL;
    struct session_data sdata;
    struct _state sstate;
@@ -175,7 +175,7 @@ int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){
             #ifdef HAVE_ANTIVIRUS
                if(cfg->use_antivirus == 1){
                   gettimeofday(&tv1, &tz);
-                  sdata.rav = do_av_check(&sdata, rctptoemail, &virusinfo[0], data, cfg);
+                  sdata.rav = do_av_check(&sdata, rcpttoemail, &virusinfo[0], data, cfg);
                   gettimeofday(&tv2, &tz);
                   sdata.__av = tvdiff(tv2, tv1);
                }
@@ -190,7 +190,7 @@ int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){
             #endif
                   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: round %d in injection", sdata.ttmpfile, i);
 
-                  extractEmail(sdata.rcptto[i], rctptoemail);
+                  extractEmail(sdata.rcptto[i], rcpttoemail);
 
                   /* copy default config to enable policy support */
                   //memcpy(&my_cfg, cfg, sizeof(struct __config));
@@ -238,9 +238,9 @@ int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){
 
                   /* set the accept buffer */
 
-                  snprintf(sdata.acceptbuf, SMALLBUFSIZE-1, "250 Ok %s <%s>\r\n", sdata.ttmpfile, rctptoemail);
+                  snprintf(sdata.acceptbuf, SMALLBUFSIZE-1, "250 Ok %s <%s>\r\n", sdata.ttmpfile, rcpttoemail);
 
-                  if(inj == ERR) snprintf(sdata.acceptbuf, SMALLBUFSIZE-1, "451 %s <%s>\r\n", sdata.ttmpfile, rctptoemail);
+                  if(inj == ERR) snprintf(sdata.acceptbuf, SMALLBUFSIZE-1, "451 %s <%s>\r\n", sdata.ttmpfile, rcpttoemail);
 
                   write1(new_sd, sdata.acceptbuf, strlen(sdata.acceptbuf), sdata.tls, data->ssl);
 
@@ -408,8 +408,11 @@ AFTER_PERIOD:
 
                /* check against blackhole addresses */
 
-               extractEmail(buf, rctptoemail);
+               extractEmail(buf, rcpttoemail);
 
+            #ifdef HAVE_MULTITENANCY
+               if(sdata.customer_id == 0) sdata.customer_id = get_customer_id_by_rcpt_to_email(rcpttoemail, data);
+            #endif
 
                if(sdata.num_of_rcpt_to < MAX_RCPT_TO-1) sdata.num_of_rcpt_to++;
 

util/db-mysql-multitenancy.sql

@@ -0,0 +1,378 @@
+
+drop table if exists `sph_counter`;
+create table if not exists `sph_counter` (
+  `counter_id` int not null,
+  `max_doc_id` int not null,
+  primary key (`counter_id`)
+);
+
+
+drop table if exists `sph_index`;
+create table if not exists `sph_index` (
+  `id` bigint not null,
+  `from` char(255) default null,
+  `to` text(8192) default null,
+  `fromdomain` char(255) default null,
+  `todomain` text(512) default null,
+  `subject` text(512) default null,
+  `arrived` int unsigned not null,
+  `sent` int unsigned not null,
+  `body` text,
+  `size` int default '0',
+  `direction` int default 0,
+  `customer` tinyint unsigned default 0,
+  `folder` int default 0,
+  `attachments` int default 0,
+  `attachment_types` text(512) default null,
+  primary key (`id`)
+) Engine=InnoDB;
+
+
+drop table if exists `metadata`;
+create table if not exists `metadata` (
+  `id` bigint unsigned not null auto_increment,
+  `from` varchar(128) not null,
+  `fromdomain` varchar(64) not null,
+  `subject` text(512) default null,
+  `spam` tinyint(1) default 0,
+  `arrived` int unsigned not null,
+  `sent` int unsigned not null,
+  `retained` int unsigned not null,
+  `deleted` tinyint(1) default 0,
+  `size` int default 0,
+  `hlen` int default 0,
+  `direction` int default 0,
+  `customer` tinyint unsigned default 0,
+  `attachments` int default 0,
+  `piler_id` char(36) not null,
+  `message_id` varchar(128) not null,
+  `reference` char(64) not null,
+  `digest` char(64) not null,
+  `bodydigest` char(64) not null,
+  `vcode` char(64) default null,
+  primary key (`id`)
+) Engine=InnoDB;
+
+create index metadata_idx on metadata(`piler_id`);
+create index metadata_idx2 on metadata(`message_id`); 
+create index metadata_idx3 on metadata(`reference`); 
+create index metadata_idx4 on metadata(`bodydigest`); 
+create index metadata_idx5 on metadata(`deleted`); 
+create index metadata_idx6 on metadata(`arrived`); 
+create index metadata_idx7 on metadata(`retained`); 
+create index metadata_idx8 on metadata(`customer`); 
+
+
+drop table if exists `rcpt`;
+create table if not exists `rcpt` (
+   `id` bigint unsigned not null,
+   `to` varchar(128) not null,
+   `todomain` varchar(64) not null,
+   unique(`id`,`to`)
+) Engine=InnoDB;
+
+create index `rcpt_idx` on `rcpt`(`id`);
+create index `rcpt_idx2` on `rcpt`(`to`);
+
+
+drop view if exists `v_messages`;
+create view `v_messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`metadata`.`fromdomain` AS `fromdomain`,`rcpt`.`to` AS `to`,`rcpt`.`todomain` AS `todomain`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`sent` AS `sent`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`);
+
+
+drop table if exists `attachment`;
+create table if not exists `attachment` (
+   `id` bigint unsigned not null auto_increment,
+   `piler_id` char(36) not null,
+   `attachment_id` int not null,
+   `name` varchar(128) default null,
+   `type` varchar(128) default null,
+   `sig` char(64) not null,
+   `size` int default 0,
+   `ptr` bigint unsigned default 0,
+   `deleted` tinyint(1) default 0,
+   primary key (`id`)
+) Engine=InnoDB;
+
+create index `attachment_idx` on `attachment`(`piler_id`);
+create index `attachment_idx2` on `attachment`(`sig`);
+create index `attachment_idx3` on `attachment`(`ptr`);
+
+drop view if exists `v_attachment`;
+create view `v_attachment` AS select `id` as `i`, `piler_id`, `attachment_id`, `ptr`, (select count(*) from `attachment` where `ptr`=`i`) as `refcount` from `attachment`;
+
+
+drop table if exists `tag`;
+create table if not exists `tag` (
+   `_id` bigint unsigned auto_increment not null,
+   `id` bigint not null,
+   `uid` int not null,
+   `tag` varchar(255) default null,
+   unique(`id`, `uid`),
+   key (`_id`)
+) ENGINE=InnoDB;
+
+
+drop table if exists `archiving_rule`;
+create table if not exists `archiving_rule` (
+   `id` bigint unsigned not null auto_increment,
+   `from` char(128) default null,
+   `to` char(255) default null,
+   `subject` char(255) default null,
+   `_size` char(2) default null,
+   `size` int default 0,
+   `attachment_type` char(128) default null,
+   `_attachment_size` char(2) default null,
+   `attachment_size` int default 0,
+   `spam` tinyint(1) default -1,
+   `days` int default 0,
+   primary key (`id`),
+   unique(`from`,`to`,`subject`,`_size`,`size`,`attachment_type`,`_attachment_size`,`attachment_size`,`spam`) 
+) ENGINE=InnoDB;
+
+
+drop table if exists `retention_rule`;
+create table if not exists `retention_rule` (
+   `id` bigint unsigned not null auto_increment,
+   `from` char(128) default null,
+   `to` char(255) default null,
+   `subject` char(255) default null,
+   `_size` char(2) default null,
+   `size` int default 0,
+   `attachment_type` char(128) default null,
+   `_attachment_size` char(2) default null,
+   `attachment_size` int default 0,
+   `spam` tinyint(1) default -1,
+   `days` int default 0,
+   primary key (`id`),
+   unique(`from`,`to`,`subject`,`_size`,`size`,`attachment_type`,`_attachment_size`,`attachment_size`,`spam`)
+) ENGINE=InnoDB;
+
+
+drop table if exists `counter`;
+create table if not exists `counter` (
+   `rcvd` bigint unsigned default 0,
+   `virus` bigint unsigned default 0,
+   `duplicate` bigint unsigned default 0,
+   `ignore` bigint unsigned default 0,
+   `size` bigint unsigned default 0
+) Engine=InnoDB;
+
+insert into `counter` values(0, 0, 0, 0, 0);
+
+
+drop table if exists `option`;
+create table if not exists `option` (
+   `key` char(64) not null,
+   `value` char(32) not null
+) Engine=InnoDB;
+
+insert into `option` (`key`, `value`) values('enable_purge', '1');
+
+
+drop table if exists `search`;
+create table if not exists `search` (
+   `email` char(128) not null,
+   `ts` int default 0,
+   `term` text(512) not null
+) Engine=InnoDB;
+
+create index `search_idx` on `search`(`email`);
+
+
+drop table if exists `user_settings`;
+create table if not exists `user_settings` (
+   `username` char(64) not null unique,
+   `pagelen` int default 20,
+   `theme` char(8) default 'default',
+   `lang` char(2) default NULL
+);
+
+create index `user_settings_idx` on `user_settings`(`username`);
+
+
+drop table if exists `user`;
+create table if not exists `user` (
+   `uid` int unsigned not null primary key,
+   `username` char(64) not null unique,
+   `realname` char(64) default null,
+   `samaccountname` char(64) default null,
+   `password` char(128) default null,
+   `domain` char(64) default null,
+   `dn` char(255) default '*',
+   `isadmin` tinyint default 0
+) Engine=InnoDB;
+
+insert into `user` (`uid`, `username`, `realname`, `samaccountname`, `password`, `isadmin`, `domain`) values (0, 'admin', 'built-in piler admin', '', '$1$PItc7d$zsUgON3JRrbdGS11t9JQW1', 1, 'local');
+
+drop table if exists `email`;
+create table if not exists `email` (
+   `uid` int unsigned not null,
+   `email` char(128) not null primary key
+) ENGINE=InnoDB;
+
+insert into `email` (`uid`, `email`) values(0, 'admin@local');
+
+
+create table if not exists `email_groups` (
+   `uid` int unsigned not null,
+   `gid` int unsigned not null,
+   unique key `uid` (`uid`,`gid`),
+   key `email_groups_idx` (`uid`,`gid`)
+) ENGINE=InnoDB;
+
+
+create table if not exists `group` (
+   `id` bigint unsigned not null auto_increment primary key,
+   `groupname` char(255) not null unique
+) ENGINE=InnoDB;
+
+
+create table if not exists `group_user` (
+   `id` bigint unsigned not null,
+   `uid` int unsigned not null,
+   key `group_user_idx` (`id`),
+   key `group_user_idx2` (`uid`)
+) ENGINE=InnoDB;
+
+
+create table if not exists `group_email` (
+   `id` bigint unsigned not null,
+   `email` char(128) not null,
+   key `group_email_idx` (`id`)
+) ENGINE=InnoDB;
+
+
+create table if not exists `domain_user` (
+   `domain` char(64) not null,
+   `uid` int unsigned not null,
+   key `domain_user_idx` (`domain`),
+   key `domain_user_idx2` (`uid`)
+) ENGINE=InnoDB;
+
+
+create table if not exists `folder` (
+   `id` int not null auto_increment,
+   `parent_id` int default 0,
+   `name` char(64) not null,
+   unique(`parent_id`, `name`),
+   primary key (`id`)
+) Engine=InnoDB;
+
+
+create table if not exists `folder_user` (
+   `id` bigint unsigned not null,
+   `uid` int unsigned not null,
+   key `folder_user_idx` (`id`),
+   key `folder_user_idx2` (`uid`)
+) ENGINE=InnoDB;
+
+
+create table if not exists `folder_extra` (
+   `id` int unsigned not null auto_increment,
+   `uid` int unsigned not null,
+   `name` char(64) not null,
+   unique(uid, name),
+   key (`id`)
+) ENGINE=InnoDB;
+
+
+create table if not exists `folder_message` (
+   folder_id bigint not null,
+   id bigint not null,
+   unique(folder_id, id)
+) ENGINE=InnoDB;
+
+
+create table if not exists `note` (
+   `_id` bigint unsigned auto_increment not null,
+   `id` bigint unsigned not null,
+   `uid` int not null,
+   `note` text default null,
+   unique(`id`, `uid`),
+   key (`_id`)
+) ENGINE=InnoDB;
+
+
+create table if not exists `remote` (
+   `remotedomain` char(64) not null primary key,
+   `remotehost` char(64) not null,
+   `basedn` char(255) not null,
+   `binddn` char(255) not null,
+   `sitedescription` char(64) default null
+) ENGINE=InnoDB;
+
+
+drop table if exists `domain`;
+create table if not exists `domain` (
+   `domain` char(64) not null primary key,
+   `mapped` char(64) not null
+) ENGINE=InnoDB;
+
+insert into `domain` (`domain`, `mapped`) values('local', 'local');
+
+
+drop table if exists `audit`;
+create table if not exists `audit` (
+   `id` bigint unsigned not null auto_increment,
+   `ts` int not null,
+   `email` varchar(128) not null,
+   `action` int not null,
+   `ipaddr` char(15) not null,
+   `meta_id` bigint unsigned not null,
+   `description` varchar(255) default null,
+   `vcode` char(64) default null,
+   primary key (`id`)
+) ENGINE=InnoDB;
+
+create index `audit_idx` on `audit`(`email`);
+create index `audit_idx2` on `audit`(`action`);
+create index `audit_idx3` on `audit`(`ipaddr`);
+create index `audit_idx4` on `audit`(`ts`);
+
+
+
+drop table if exists `google`;
+create table if not exists `google` (
+   `id` bigint unsigned not null primary key,
+   `email` char(128) not null unique,
+   `access_token` char(255) default null,
+   `refresh_token` char(255) default null,
+   `created` int default 0
+) ENGINE=InnoDB;
+
+
+drop table if exists `google_imap`;
+create table if not exists `google_imap` (
+   `id` bigint unsigned not null,
+   `email` char(128) not null,
+   `last_msg_id` bigint default 0,
+   key(`email`)
+) ENGINE=InnoDB;
+
+
+CREATE TABLE IF NOT EXISTS `counter_stats` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `date` int(11) NOT NULL,
+  `email` varchar(255) NOT NULL,
+  `domain` varchar(255) NOT NULL,
+  `sent` int(11) NOT NULL,
+  `recd` int(11) NOT NULL,
+  `sentsize` int(11) NOT NULL,
+  `recdsize` int(11) NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `date` (`date`),
+  KEY `email` (`email`),
+  KEY `domain` (`domain`)
+) ENGINE=InnoDB;
+
+
+CREATE TABLE IF NOT EXISTS `customer` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `customername` varchar(255) NOT NULL,
+  `customerdesc` text NOT NULL,
+  `customerpoc` int(11) NOT NULL,
+  `guid` varchar(255) NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `guid` (`guid`)
+) ENGINE=InnoDB;
+