experimenting with zstd

Change-Id: Ifa902b7d5bd260157846a41c745ace0268e4b51b
Signed-off-by: SJ <sj@acts.hu>

configure

@@ -3443,6 +3443,7 @@ have_psql="no"
 have_tre="no"
 have_zip="no"
 have_zlib="no"
+have_zstd="no"
 have_tcpwrappers="no"
 have_tweak_sent_time="no"
 
@@ -4146,6 +4147,96 @@ fi
 ac_cv_lib_zip=ac_cv_lib_zip_main
 
 
+for ac_header in zstd.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "zstd.h" "ac_cv_header_zstd_h" "$ac_includes_default"
+if test "x$ac_cv_header_zstd_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_ZSTD_H 1
+_ACEOF
+ have_zstd=yes
+else
+  have_zstd=no
+fi
+
+done
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lzstd" >&5
+$as_echo_n "checking for main in -lzstd... " >&6; }
+if ${ac_cv_lib_zstd_main+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lzstd  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+return main ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_zstd_main=yes
+else
+  ac_cv_lib_zstd_main=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_main" >&5
+$as_echo "$ac_cv_lib_zstd_main" >&6; }
+if test "x$ac_cv_lib_zstd_main" = xyes; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD_compress in -lzstd" >&5
+$as_echo_n "checking for ZSTD_compress in -lzstd... " >&6; }
+if ${ac_cv_lib_zstd_ZSTD_compress+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lzstd  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ZSTD_compress ();
+int
+main ()
+{
+return ZSTD_compress ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_zstd_ZSTD_compress=yes
+else
+  ac_cv_lib_zstd_ZSTD_compress=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_compress" >&5
+$as_echo "$ac_cv_lib_zstd_ZSTD_compress" >&6; }
+if test "x$ac_cv_lib_zstd_ZSTD_compress" = xyes; then :
+  have_zstd=yes
+else
+  echo "libzstd.so is not found"; have_zstd=no
+fi
+
+fi
+ac_cv_lib_z=ac_cv_lib_zstd_main
+
 
 for ac_header in zlib.h
 do :
@@ -4716,6 +4807,16 @@ if test "$have_tre" = "yes"; then
    antispam_libs="$antispam_libs -ltre"
 fi
 
+if test "$have_zstd" = "yes"; then
+   echo "zstd library: yes"
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_ZSTD 1
+_ACEOF
+
+   antispam_libs="$antispam_libs -lzstd"
+fi
+
 if test "$have_zip" = "yes"; then
    echo "zip library: yes"
 

configure.in

@@ -41,6 +41,7 @@ have_psql="no"
 have_tre="no"
 have_zip="no"
 have_zlib="no"
+have_zstd="no"
 have_tcpwrappers="no"
 have_tweak_sent_time="no"
 
@@ -169,6 +170,10 @@ dnl libzip
 AC_CHECK_HEADERS(zip.h, have_zip=yes, echo "zip.h is not found")
 AC_CHECK_LIB([zip],[main],[AC_CHECK_LIB(zip, zip_open, have_zip=yes, echo "libzip.so is not found"; have_zip=no)],[],[])ac_cv_lib_zip=ac_cv_lib_zip_main
 
+dnl check zstd library
+
+AC_CHECK_HEADERS(zstd.h, have_zstd=yes, have_zstd=no)
+AC_CHECK_LIB([zstd],[main],[AC_CHECK_LIB(zstd, ZSTD_compress, have_zstd=yes, echo "libzstd.so is not found"; have_zstd=no)],[],[])ac_cv_lib_z=ac_cv_lib_zstd_main
 
 dnl zlib
 
@@ -434,6 +439,12 @@ if test "$have_tre" = "yes"; then
    antispam_libs="$antispam_libs -ltre"
 fi
 
+if test "$have_zstd" = "yes"; then
+   echo "zstd library: yes"
+   AC_DEFINE_UNQUOTED(HAVE_ZSTD, 1, [libzstd support])
+   antispam_libs="$antispam_libs -lzstd"
+fi
+
 if test "$have_zip" = "yes"; then
    echo "zip library: yes"
    AC_DEFINE_UNQUOTED(HAVE_ZIP, 1, [libzip support])

piler-config.h.in

@@ -26,6 +26,8 @@
 
 #undef HAVE_LIBWRAP
 
+#undef HAVE_ZSTD
+
 #undef HAVE_TWEAK_SENT_TIME
 
 #undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT

src/store.c

@@ -13,8 +13,12 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <syslog.h>
+#ifdef HAVE_ZSTD
+   #include <zstd.h>
+#else
+   #include <zlib.h>
+#endif
 #include <piler.h>
-#include <zlib.h>
 #include <openssl/blowfish.h>
 #include <openssl/evp.h>
 #include <errno.h>
@@ -43,8 +47,13 @@ int store_file(struct session_data *sdata, char *filename, int len, struct __con
    int ret=0, rc, fd, n;
    char *addr, *p, *p0, *p1, *p2, s[SMALLBUFSIZE];
    struct stat st;
-   Bytef *z=NULL;
+#ifdef HAVE_ZSTD
+   void *dst;
+   size_t dstlen;
+#else
+   Bytef *dst=NULL;
    uLongf dstlen;
+#endif
 
    EVP_CIPHER_CTX ctx;
    unsigned char *outbuf=NULL;
@@ -80,23 +89,43 @@ int store_file(struct session_data *sdata, char *filename, int len, struct __con
 
    if(addr == MAP_FAILED) return ret;
 
+   rc = OK;
+
+#ifdef HAVE_ZSTD
+   dstlen = ZSTD_compressBound(len);
+   dst = malloc(dstlen);
+
+   if(dst == NULL){
+      munmap(addr, len);
+      syslog(LOG_PRIORITY, "%s: cannot malloc for zstd buffer", sdata->ttmpfile);
+      return ret;
+   }
+
+   size_t const cSize = ZSTD_compress(dst, destlen, addr, len, 1);
+   if(ZSTD_isError(cSize)){
+      syslog(LOG_PRIORITY, "%s: error zstd compressing: %s", sdata->ttmpfile, ZSTD_getErrorName(cSize));
+      rc = ERR;
+   }
+#else
    dstlen = compressBound(len);
+   dst = malloc(dstlen);
 
-   z = malloc(dstlen);
+   if(dst == NULL){
-
-   if(z == NULL){
       munmap(addr, len);
       syslog(LOG_PRIORITY, "%s: cannot malloc for z buffer", sdata->ttmpfile);
       return ret;
    }
 
-   rc = compress(z, &dstlen, (const Bytef *)addr, len);
+   if(compress(dst, &dstlen, (const Bytef *)addr, len) != Z_OK)
+      rc = ERR;
+#endif
+
    gettimeofday(&tv2, &tz);
    sdata->__compress += tvdiff(tv2, tv1);
 
    munmap(addr, len);
 
-   if(rc != Z_OK) goto ENDE;
+   if(rc == ERR) goto ENDE;
 
    if(cfg->encrypt_messages == 1){
       gettimeofday(&tv1, &tz);
@@ -107,7 +136,7 @@ int store_file(struct session_data *sdata, char *filename, int len, struct __con
       outbuf = malloc(dstlen + EVP_MAX_BLOCK_LENGTH);
       if(outbuf == NULL) goto ENDE;
 
-      if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, z, dstlen)) goto ENDE;
+      if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, dst, dstlen)) goto ENDE;
       if(!EVP_EncryptFinal_ex(&ctx, outbuf + outlen, &tmplen)) goto ENDE;
       outlen += tmplen;
       EVP_CIPHER_CTX_cleanup(&ctx);
@@ -161,7 +190,7 @@ int store_file(struct session_data *sdata, char *filename, int len, struct __con
       writelen = outlen;
    }
    else {
-      n = write(fd, z, dstlen);
+      n = write(fd, dst, dstlen);
       writelen = dstlen;
    }
 
@@ -185,7 +214,7 @@ int store_file(struct session_data *sdata, char *filename, int len, struct __con
 
 ENDE:
    if(outbuf) free(outbuf);
-   if(z) free(z);
+   if(dst) free(dst);
 
    return ret;
 }