added a few parser tests

configure

@@ -3423,7 +3423,7 @@ os=`uname -s`
 
 id_bin="id"
 
-SUBDIRS="src etc util init.d"
+SUBDIRS="src etc util init.d test"
 
 
 
@@ -4153,7 +4153,7 @@ CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
 OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o list.o parser.o parser_utils.o rules.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o $objs"
 
-ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile"
+ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile test/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -4863,6 +4863,7 @@ do
     "etc/Makefile") CONFIG_FILES="$CONFIG_FILES etc/Makefile" ;;
     "util/Makefile") CONFIG_FILES="$CONFIG_FILES util/Makefile" ;;
     "init.d/Makefile") CONFIG_FILES="$CONFIG_FILES init.d/Makefile" ;;
+    "test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;;
 
   *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
   esac

configure.in

@@ -51,7 +51,7 @@ os=`uname -s`
 id_bin="id"
 
 dnl SUBDIRS="src etc util perl init.d templates history contrib/stat"
-SUBDIRS="src etc util init.d"
+SUBDIRS="src etc util init.d test"
 
 
 dnl static build
@@ -276,6 +276,6 @@ CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
 OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o list.o parser.o parser_utils.o rules.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o $objs"
 
-AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile])
+AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile test/Makefile])
 AC_OUTPUT
 

src/config.h

@@ -13,7 +13,7 @@
 
 #define VERSION "0.1.19"
 
-#define BUILD 672
+#define BUILD 675
 
 #define HOSTID "mailarchiver"
 

src/parser_utils.c

@@ -460,6 +460,9 @@ int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct _state *state){
 
 void translateLine(unsigned char *p, struct _state *state){
    int url=0;
+   int has_url = 0;
+
+   if(strcasestr((char *)p, "http://") || strcasestr((char *)p, "https://")) has_url = 1;
 
    for(; *p; p++){
 
@@ -471,11 +474,13 @@ void translateLine(unsigned char *p, struct _state *state){
 
       if(*p == '.' || *p == '-'){ continue; }
 
-      if(strncasecmp((char *)p, "http://", 7) == 0){ p += 7; url = 1; continue; }
+      if(has_url == 1){
-      if(strncasecmp((char *)p, "https://", 8) == 0){ p += 8; url = 1; continue; }
+         if(strncasecmp((char *)p, "http://", 7) == 0){ p += 7; url = 1; continue; }
+         if(strncasecmp((char *)p, "https://", 8) == 0){ p += 8; url = 1; continue; }
 
-      if(url == 1 && (*p == '.' || *p == '-' || *p == '_' || *p == '/' || isalnum(*p)) ) continue;
+         if(url == 1 && (*p == '.' || *p == '-' || *p == '_' || *p == '/' || isalnum(*p)) ) continue;
-      if(url == 1) url = 0;
+         if(url == 1) url = 0;
+      }
 
       if(state->texthtml == 1 && state->message_state == MSG_BODY && strncmp((char *)p, "HTML*", 5) == 0){
          p += 5;
@@ -484,12 +489,8 @@ void translateLine(unsigned char *p, struct _state *state){
          }
       }
 
-      if(delimiter_characters[(unsigned int)*p] != ' ')
+      if(delimiter_characters[(unsigned int)*p] != ' ') *p = ' ';
-         *p = ' ';
+      /* we MUSTN'T convert it to lowercase in the 'else' case, because it breaks utf-8 encoding! */
-      else {
-         // commented out because it breaks utf-8 encoding, 2011.12.07.
-         //*p = tolower(*p);
-      }
 
    }
 
@@ -585,8 +586,11 @@ void degenerateToken(unsigned char *p){
 
 
 void fixURL(char *url){
+   int len=0;
    char *p, *q, fixed_url[SMALLBUFSIZE];
 
+   if(strlen(url) < 3) return;
+
    memset(fixed_url, 0, sizeof(fixed_url));
 
    p = url;
@@ -600,6 +604,12 @@ void fixURL(char *url){
    snprintf(fixed_url, sizeof(fixed_url)-1, "__URL__%s ", p);
    fix_email_address_for_sphinx(fixed_url+7);
 
+   len = strlen(fixed_url);
+   if(len > 9 && fixed_url[len-2] == 'X'){
+      fixed_url[len-2] = ' ';
+      fixed_url[len-1] = '\0';
+   }
+
    strcpy(url, fixed_url);   
 }
 

src/pilerimport.c

@@ -168,7 +168,7 @@ void usage(){
 
 
 int main(int argc, char **argv){
-   int i, rc;
+   int i, rc=0;
    char *configfile=CONFIG_FILE, *mailbox=NULL, *emlfile=NULL, *directory=NULL;
    char *imapserver=NULL, *username=NULL, *password=NULL;
    struct session_data sdata;