improved pilerimport to support sphinx queries

src/config.h

@@ -14,7 +14,7 @@
 
 #define VERSION "1.2.0-master"
 
-#define BUILD 917
+#define BUILD 918
 
 #define HOSTID "mailarchiver"
 

src/pilerexport.c

@@ -21,10 +21,17 @@ extern char *optarg;
 extern int optind;
 
 int dryrun = 0;
+int exportall = 0;
+int rc = 0;
 char *query=NULL;
+int verbosity = 0;
+int max_matches = 1000;
+char *index_list = "main1,dailydelta1,delta1";
 regex_t regexp;
 
 
+int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg);
+
 
 void usage(){
    printf("\nusage: pilerexport \n\n");
@@ -33,6 +40,9 @@ void usage(){
    printf("    -f|--from <email@address> -r|--to <email@address>\n");
    printf("    -F|--from-domain <domain.com> -R|--to-domain <domain.com>\n");
    printf("    -s|--minsize <number> -S|--maxsize <number>\n");
+   printf("    -w|--where-condition (eg. \"match('@subject: piler')\"\n");
+   printf("    -m|--max-matches (default: %d)\n", max_matches);
+   printf("    -i|--index-list (default: %s)\n", index_list);
    printf("    -A|--all  -d|--dryrun \n");
    printf("\n    use -A if you don't want to specify the start/stop time nor any from/to address\n\n");
 
@@ -136,229 +146,92 @@ int append_string_to_buffer(char **buffer, char *str){
 }
 
 
-int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg){
+uint64 run_query(struct session_data *sdata, struct session_data *sdata2, struct __data *data, char *where_condition, uint64 last_id, int *num, struct __config *cfg){
-   FILE *f;
+   MYSQL_RES *res;
-   uint64 id, n=0;
+   MYSQL_ROW row;
-   char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
-   char filename[SMALLBUFSIZE];
    int rc=0;
+   uint64 id=0;
+   char s[SMALLBUFSIZE];
 
+   *num = 0;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), s, cfg) == ERR) return ERR;
+   if(!where_condition) return id;
 
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
+   rc += append_string_to_buffer(&query, s);
 
-   p_bind_init(data);
+   snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);
 
-   if(p_exec_query(sdata, data->stmt_generic, data) == ERR) goto ENDE;
+   if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
+      res = mysql_store_result(&(sdata2->mysql));
+      if(res != NULL){
+         while((row = mysql_fetch_row(res))){
+            id = strtoull(row[0], NULL, 10);
+            (*num)++;
+            rc += append_string_to_buffer(&query, row[0]);
+            rc += append_string_to_buffer(&query, ",");
 
-
-
-   p_bind_init(data);
-
-   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
-   data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;
-   data->sql[data->pos] = &digest[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(digest)-2; data->pos++;
-   data->sql[data->pos] = &bodydigest[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(bodydigest)-2; data->pos++;
-
-   p_store_results(sdata, data->stmt_generic, data);
-
-   while(p_fetch_results(data->stmt_generic) == OK){
-
-      if(id > 0){
-
-         if(dryrun == 0){
-
-            snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
-
-            f = fopen(filename, "w");
-            if(f){
-               rc = retrieve_email_from_archive(sdata, data, f, cfg);
-               fclose(f);
-
-               n++;
-
-               snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-
-               make_digests(sdata, cfg);
-
-               if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
-                  printf("exported: %10llu\r", n); fflush(stdout);
-               }
-               else
-                  printf("verification FAILED. %s\n", filename);
-
-            }
-            else printf("cannot open: %s\n", filename);
-         }
-         else {
-            printf("id:%llu\n", id);
          }
+         mysql_free_result(res);
 
+         rc += append_string_to_buffer(&query, "-1)");
       }
-
    }
 
-   p_free_results(data->stmt_generic);
+   if(!rc) export_emails_matching_to_query(sdata, data, query, cfg);
 
-ENDE:
+   free(query);
-   close_prepared_statement(data->stmt_generic);
+   query = NULL;
 
 
-   printf("\n");
+   return id;
-
-   return rc;
 }
 
 
-int main(int argc, char **argv){
+uint64 get_total_found(struct session_data *sdata){
-   int c, rc, exportall=0, minsize=0, maxsize=0;
+   MYSQL_RES *res;
-   int where_condition=0;
+   MYSQL_ROW row;
-   size_t nmatch=0;
+   uint64 total_found=0;
-   unsigned long startdate=0, stopdate=0;
-   char *configfile=CONFIG_FILE;
-   char *to=NULL, *from=NULL, *todomain=NULL, *fromdomain=NULL;
-   char s[SMALLBUFSIZE];
-   struct session_data sdata;
-   struct __data data;
-   struct __config cfg;
 
-
+   if(mysql_real_query(&(sdata->mysql), "SHOW META LIKE 'total_found'", 28) == 0){
-   if(regcomp(&regexp, "^([\\+a-z0-9_\\.@\\-]+)$", REG_ICASE | REG_EXTENDED)){
+      res = mysql_store_result(&(sdata->mysql));
-      p_clean_exit("cannot compile rule!", 1);
+      if(res != NULL){
-   }
+         while((row = mysql_fetch_row(res))){
-
+            total_found = strtoull(row[1], NULL, 10);
-
+         }
-   while(1){
+         mysql_free_result(res);
-
-#ifdef _GNU_SOURCE
-      static struct option long_options[] =
-         {
-            {"config",       required_argument,  0,  'c' },
-            {"minsize",      required_argument,  0,  's' },
-            {"maxsize",      required_argument,  0,  'S' },
-            {"all",          no_argument,        0,  'A' },
-            {"dry-run",      no_argument,        0,  'd' },
-            {"dryrun",       no_argument,        0,  'd' },
-            {"help",         no_argument,        0,  'h' },
-            {"version",      no_argument,        0,  'v' },
-            {"from",         required_argument,  0,  'f' },
-            {"to",           required_argument,  0,  'r' },
-            {"from-domain",  required_argument,  0,  'F' },
-            {"to-domain",    required_argument,  0,  'R' },
-            {"start-date",   required_argument,  0,  'a' },
-            {"stop-date",    required_argument,  0,  'b' },
-            {"id",           required_argument,  0,  'i' },
-            {0,0,0,0}
-         };
-
-      int option_index = 0;
-
-      c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:i:Adhv?", long_options, &option_index);
-#else
-      c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:i:Adhv?");
-#endif
-
-      if(c == -1) break;
-
-      switch(c){
-
-         case 'c' :
-                    configfile = optarg;
-                    break;
-
-         case 's' :
-                    minsize = atoi(optarg);
-                    break;
-
-         case 'S' :
-                    maxsize = atoi(optarg);
-                    break;
-
-
-         case 'A' :
-                    exportall = 1;
-                    break;
-
-         case 'f' :
-
-                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
-                       printf("%s is not a valid email address\n", optarg);
-                       break;
-                    }
-
-                    rc = append_email_to_buffer(&from, optarg);
-
-                    break;
-
-         case 'r' :
-
-                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
-                       printf("%s is not a valid email address\n", optarg);
-                       break;
-                    }
-
-                    rc = append_email_to_buffer(&to, optarg);
-
-                    break;
-
-         case 'F' :
-
-                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
-                       printf("%s is not a valid domain name\n", optarg);
-                       break;
-                    }
-
-                    rc = append_email_to_buffer(&fromdomain, optarg);
-
-                    break;           
-
-         case 'R' :
-
-                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
-                       printf("%s is not a valid domain name\n", optarg);
-                       break;
-                    }
-
-                    rc = append_email_to_buffer(&todomain, optarg);
-
-                    break;                     
-                    
-         case 'a' :
-                    startdate = convert_time(optarg, 0, 0, 0);
-                    break;
-
-
-         case 'b' :
-                    stopdate = convert_time(optarg, 23, 59, 59);
-                    break;
-
-
-         case 'd' :
-                    dryrun = 1;
-                    break;
-
-
-         default  :
-                    usage();
-                    break;
       }
-
    }
 
-
+   return total_found;
-   if(from == NULL && to == NULL && fromdomain == NULL && todomain == NULL && startdate == 0 && stopdate == 0 && exportall == 0) usage();
+}
 
 
-   regfree(&regexp);
+void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, struct __data *data, char *where_condition, struct __config *cfg){
+   int n;
+   uint64 count=0, last_id=0, total_found=0;
 
-   if(!can_i_write_current_directory()) __fatal("cannot write current directory!");
+   last_id = run_query(sdata, sdata2, data, where_condition, last_id, &n, cfg);
+   count += n;
 
-   (void) openlog("pilerexport", LOG_PID, LOG_MAIL);
+   total_found = get_total_found(sdata2);
+
+   while(count < total_found){
+      last_id = run_query(sdata, sdata2, data, where_condition, last_id, &n, cfg);
+      count += n;
+   }
+
+}
+
+
+int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
+   int where_condition=0;
+   char s[SMALLBUFSIZE];
 
    if(exportall == 1){
       rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM ");
       rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
-      goto GO;
+      return rc;
    }
 
    snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE ", SQL_MESSAGES_VIEW);
@@ -375,7 +248,6 @@ int main(int argc, char **argv){
       where_condition++;
    }
 
-
    if(to){
       if(where_condition) rc = append_string_to_buffer(&query, " AND ");
 
@@ -451,9 +323,238 @@ int main(int argc, char **argv){
 
    rc += append_string_to_buffer(&query, " ORDER BY id ASC");
 
+   return rc;
+}
 
-GO:
+
-   if(rc) p_clean_exit("malloc problem building query", 1);
+int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg){
+   FILE *f;
+   uint64 id, n=0;
+   char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
+   char filename[SMALLBUFSIZE];
+   int rc=0;
+
+
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), s, cfg) == ERR) return ERR;
+
+
+   p_bind_init(data);
+
+   if(p_exec_query(sdata, data->stmt_generic, data) == ERR) goto ENDE;
+
+
+
+   p_bind_init(data);
+
+   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+   data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;
+   data->sql[data->pos] = &digest[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(digest)-2; data->pos++;
+   data->sql[data->pos] = &bodydigest[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(bodydigest)-2; data->pos++;
+
+   p_store_results(sdata, data->stmt_generic, data);
+
+   while(p_fetch_results(data->stmt_generic) == OK){
+
+      if(id > 0){
+
+         if(dryrun == 0){
+
+            snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
+
+            f = fopen(filename, "w");
+            if(f){
+               rc = retrieve_email_from_archive(sdata, data, f, cfg);
+               fclose(f);
+
+               n++;
+
+               snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
+
+               make_digests(sdata, cfg);
+
+               if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
+                  printf("exported: %10llu\r", n); fflush(stdout);
+               }
+               else
+                  printf("verification FAILED. %s\n", filename);
+
+            }
+            else printf("cannot open: %s\n", filename);
+         }
+         else {
+            printf("id:%llu\n", id);
+         }
+
+      }
+
+   }
+
+   p_free_results(data->stmt_generic);
+
+ENDE:
+   close_prepared_statement(data->stmt_generic);
+
+
+   printf("\n");
+
+   return rc;
+}
+
+
+int main(int argc, char **argv){
+   int c, minsize=0, maxsize=0;
+   size_t nmatch=0;
+   unsigned long startdate=0, stopdate=0;
+   char *configfile=CONFIG_FILE;
+   char *to=NULL, *from=NULL, *todomain=NULL, *fromdomain=NULL, *where_condition=NULL;
+   struct session_data sdata, sdata2;
+   struct __data data;
+   struct __config cfg;
+
+
+   if(regcomp(&regexp, "^([\\+a-z0-9_\\.@\\-]+)$", REG_ICASE | REG_EXTENDED)){
+      p_clean_exit("cannot compile rule!", 1);
+   }
+
+
+   while(1){
+
+#ifdef _GNU_SOURCE
+      static struct option long_options[] =
+         {
+            {"config",       required_argument,  0,  'c' },
+            {"minsize",      required_argument,  0,  's' },
+            {"maxsize",      required_argument,  0,  'S' },
+            {"all",          no_argument,        0,  'A' },
+            {"dry-run",      no_argument,        0,  'd' },
+            {"dryrun",       no_argument,        0,  'd' },
+            {"help",         no_argument,        0,  'h' },
+            {"version",      no_argument,        0,  'v' },
+            {"from",         required_argument,  0,  'f' },
+            {"to",           required_argument,  0,  'r' },
+            {"from-domain",  required_argument,  0,  'F' },
+            {"to-domain",    required_argument,  0,  'R' },
+            {"start-date",   required_argument,  0,  'a' },
+            {"stop-date",    required_argument,  0,  'b' },
+            {"where-condition", required_argument,  0,  'w' },
+            {"max-matches",  required_argument,  0,  'm' },
+            {"index-list",   required_argument,  0,  'i' },
+            {0,0,0,0}
+         };
+
+      int option_index = 0;
+
+      c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?", long_options, &option_index);
+#else
+      c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?");
+#endif
+
+      if(c == -1) break;
+
+      switch(c){
+
+         case 'c' :
+                    configfile = optarg;
+                    break;
+
+         case 's' :
+                    minsize = atoi(optarg);
+                    break;
+
+         case 'S' :
+                    maxsize = atoi(optarg);
+                    break;
+
+
+         case 'A' :
+                    exportall = 1;
+                    break;
+
+         case 'f' :
+
+                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
+                       printf("%s is not a valid email address\n", optarg);
+                       break;
+                    }
+
+                    rc = append_email_to_buffer(&from, optarg);
+
+                    break;
+
+         case 'r' :
+
+                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
+                       printf("%s is not a valid email address\n", optarg);
+                       break;
+                    }
+
+                    rc = append_email_to_buffer(&to, optarg);
+
+                    break;
+
+         case 'F' :
+
+                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
+                       printf("%s is not a valid domain name\n", optarg);
+                       break;
+                    }
+
+                    rc = append_email_to_buffer(&fromdomain, optarg);
+
+                    break;           
+
+         case 'R' :
+
+                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
+                       printf("%s is not a valid domain name\n", optarg);
+                       break;
+                    }
+
+                    rc = append_email_to_buffer(&todomain, optarg);
+
+                    break;                     
+                    
+         case 'a' :
+                    startdate = convert_time(optarg, 0, 0, 0);
+                    break;
+
+
+         case 'b' :
+                    stopdate = convert_time(optarg, 23, 59, 59);
+                    break;
+
+         case 'w' :
+                    where_condition = optarg;
+                    break;
+
+         case 'm' :
+                    max_matches = atoi(optarg);
+                    break;
+
+         case 'i' :
+                    index_list = optarg;
+                    break;
+
+         case 'd' :
+                    dryrun = 1;
+                    break;
+
+         default  :
+                    usage();
+                    break;
+      }
+
+   }
+
+
+   if(from == NULL && to == NULL && fromdomain == NULL && todomain == NULL && where_condition == NULL && startdate == 0 && stopdate == 0 && exportall == 0) usage();
+
+
+   regfree(&regexp);
+
+   if(!can_i_write_current_directory()) __fatal("cannot write current directory!");
+
+   (void) openlog("pilerexport", LOG_PID, LOG_MAIL);
 
 
    cfg = read_config(configfile);
@@ -470,11 +571,27 @@ GO:
    }
 
 
+   if(where_condition){
 
-   rc = export_emails_matching_to_query(&sdata, &data, query, &cfg);
+      init_session_data(&sdata2, &cfg);
 
-   free(query);
+      strcpy(cfg.mysqlhost, "127.0.0.1");
+      cfg.mysqlport = 9306;
+      cfg.mysqlsocket[0] = '\0';
 
+      if(open_database(&sdata2, &cfg) == ERR){
+         p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
+      }
+
+      export_emails_matching_id_list(&sdata, &sdata2, &data, where_condition, &cfg);
+
+      close_database(&sdata2);
+   }
+   else {
+      if(build_query_from_args(from, to, fromdomain, todomain, minsize, maxsize, startdate, stopdate) > 0) p_clean_exit("malloc problem building query", 1);
+      export_emails_matching_to_query(&sdata, &data, query, &cfg);
+      free(query);
+   }
 
    close_database(&sdata);