Fixed docker config

Signed-off-by: Janos SUTO <sj@acts.hu>

docker/Dockerfile

@@ -2,48 +2,40 @@ FROM ubuntu:20.04
 
 ARG PACKAGE
 
-LABEL description="piler container" \
+LABEL description="piler ubuntu focal image" \
       maintainer="Janos SUTO, sj@acts.hu" \
       package="${PACKAGE}"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
-    DISTRO="bionic" \
+    DISTRO="focal" \
     DOWNLOAD_URL="https://download.mailpiler.com" \
     PILER_USER="piler" \
-    MYSQL_HOSTNAME="localhost" \
     MYSQL_DATABASE="piler" \
-    MYSQL_PILER_PASSWORD="piler123" \
+    SPHINX_BIN_TARGZ="sphinx-3.3.1-bin.tar.gz"
-    MYSQL_ROOT_PASSWORD="abcde123" \
-    SPHINX_BIN_TARGZ="sphinx-3.1.1-bin.tar.gz"
 
-ADD "https://bitbucket.org/jsuto/piler/downloads/${PACKAGE}" "/${PACKAGE}"
+COPY ${PACKAGE} /
-COPY start.sh /start.sh
 
 RUN apt-get update && \
     apt-get -y --no-install-recommends install \
-       wget rsyslog openssl sysstat php7.2-cli php7.2-cgi php7.2-mysql php7.2-fpm php7.2-zip php7.2-ldap \
+       wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
-       php7.2-gd php7.2-curl php7.2-xml catdoc unrtf poppler-utils nginx tnef sudo libodbc1 libpq5 libzip4 \
+       php7.4-gd php7.4-curl php7.4-xml php7.4-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip5 \
-       libtre5 libwrap0 cron libmariadb3 libmysqlclient-dev python python-mysqldb mariadb-server && \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.3 python3 python3-mysqldb && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
-    service mysql start && mysqladmin -u root password ${MYSQL_ROOT_PASSWORD} && \
     wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \
     tar zxvf ${SPHINX_BIN_TARGZ} && \
-    rm -f ${SPHINX_BIN_TARGZ} && \
     sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
     sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
-    mkdir /etc/piler && \
+    dpkg -i ${PACKAGE} && \
-    printf "[mysql]\nuser = piler\npassword = %s\n" ${MYSQL_PILER_PASSWORD} > /etc/piler/.my.cnf && \
+    ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
-    printf "[mysql]\nuser = root\npassword = %s\n" ${MYSQL_ROOT_PASSWORD} > /root/.my.cnf && \
+    rm -f ${PACKAGE} ${SPHINX_BIN_TARGZ} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
-    echo "alias mysql='mysql --defaults-file=/etc/piler/.my.cnf'" > /root/.bashrc && \
+    crontab -u $PILER_USER /usr/share/piler/piler.cron
-    echo "alias t='tail -f /var/log/syslog'" >> /root/.bashrc && \
+
-    dpkg -i $PACKAGE && \
+VOLUME ["/etc/piler"]
-    crontab -u $PILER_USER /usr/share/piler/piler.cron && \
+VOLUME ["/var/piler"]
-    touch /var/log/mail.log && \
-    rm -f $PACKAGE /etc/nginx/sites-enabled/default && \
-    sed -i 's/#ngram/ngram/g' /etc/piler/sphinx.conf.dist && \
-    sed -i 's/220/311/g' /etc/piler/sphinx.conf.dist
 
 EXPOSE 25 80 443
-VOLUME ["/var/piler"]
+
+COPY start.sh /start.sh
+
 CMD ["/start.sh"]

docker/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+IMAGE_NAME="sutoj/piler:1.3.10"
+
+if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
+
+docker build --pull --build-arg PACKAGE="$1" -t "$IMAGE_NAME" .

docker/docker-compose.yaml

@@ -0,0 +1,61 @@
+version: "3"
+services:
+
+  mysql:
+    image: mariadb:10.4
+    restart: unless-stopped
+    cap_drop:
+      - ALL
+    cap_add:
+      - dac_override
+      - setuid
+      - setgid
+    environment:
+      - MYSQL_ROOT_PASSWORD=abcde123
+    volumes:
+      - db_data:/var/lib/mysql
+
+  memcached:
+    image: memcached:latest
+    restart: unless-stopped
+    cap_drop:
+      - ALL
+    command: -m 64
+
+  piler:
+    image: sutoj/piler:1.3.10
+    init: true
+    environment:
+      - MYSQL_PILER_PASSWORD=piler123
+      - MYSQL_ROOT_PASSWORD=abcde123
+      - MYSQL_HOSTNAME=mysql
+      - PILER_HOSTNAME=archive.yourdomain.com
+      - MEMCACHED_HOST=memcached
+    ports:
+      - "25:25"
+      - "80:80"
+      - "443:443"
+    volumes:
+      - piler_etc:/etc/piler
+      - piler_var:/var/piler
+    healthcheck:
+      test: curl -s smtp://localhost/
+      interval: "60s"
+      timeout: "3s"
+      start_period: "15s"
+      retries: 3
+    deploy:
+      resources:
+        reservations:
+          memory: 512M
+        limits:
+          memory: 512M
+
+    depends_on:
+      - "memcached"
+      - "mysql"
+
+volumes:
+  db_data: {}
+  piler_etc: {}
+  piler_var: {}

docker/start.sh

@@ -4,91 +4,199 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-DATAROOTDIR="/usr/share"
+CONFIG_DIR="/etc/piler"
-SYSCONFDIR="/etc"
+VOLUME_DIR="/var/piler"
-SPHINXCFG="/etc/piler/sphinx.conf"
+PILER_CONF="${CONFIG_DIR}/piler.conf"
-PILER_HOST=${PILER_HOST:-archive.yourdomain.com}
+PILER_KEY="${CONFIG_DIR}/piler.key"
-PILER_CONF="/etc/piler/piler.conf"
+PILER_PEM="${CONFIG_DIR}/piler.pem"
-CONFIG_SITE_PHP="/etc/piler/config-site.php"
+PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
-CONFIG_PHP="/var/piler/www/config.php"
+SPHINX_CONF="${CONFIG_DIR}/sphinx.conf"
+CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
+PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
+ROOT_MY_CNF="/root/.my.cnf"
 
-create_mysql_db() {
-   echo "Creating mysql database"
 
-   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%g" \
+error() {
-       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%g" \
+   echo "ERROR:" "$*" 1>&2
-       -e "s%MYSQL_USERNAME%${PILER_USER}%g" \
+   exit 1
-       -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" \
-       "${DATAROOTDIR}/piler/db-mysql-root.sql.in" | \
-       mysql -h "$MYSQL_HOSTNAME" -u root --password="$MYSQL_ROOT_PASSWORD"
-
-   mysql -h "$MYSQL_HOSTNAME" -u "$PILER_USER" --password="$MYSQL_PILER_PASSWORD" "$MYSQL_DATABASE" < "${DATAROOTDIR}/piler/db-mysql.sql"
-
-   echo "Done."
 }
 
 
-pre_seed_sphinx() {
+log() {
-   echo "Writing sphinx configuration"
+   echo "DEBUG:" "$*"
-   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
+}
-       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
-       -e "s%MYSQL_USERNAME%${PILER_USER}%" \
-       -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" \
-       -e "s%220%311%" \
-       -e "s%type = mysql%type = mysql\n   sql_sock = /var/run/mysqld/mysqld.sock%" \
-       "${SYSCONFDIR}/piler/sphinx.conf.dist" > "$SPHINXCFG"
 
-   echo "Done."
 
-   echo "Initializing sphinx indices"
+pre_flight_check() {
-   su "$PILER_USER" -c "indexer --all --config ${SYSCONFDIR}/piler/sphinx.conf"
+   [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
-   echo "Done."
+   [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
+   [[ -v MYSQL_PILER_PASSWORD ]] || error "Missing MYSQL_PILER_PASSWORD env variable"
+   [[ -v MYSQL_ROOT_PASSWORD ]] || error "Missing MYSQL_ROOT_PASSWORD env variable"
+}
+
+
+give_it_to_piler() {
+   local f="$1"
+
+   [[ -f "$f" ]] || error "${f} does not exist, aborting"
+
+   chown "${PILER_USER}:${PILER_USER}" "$f"
+   chmod 600 "$f"
+}
+
+
+make_certificate() {
+   local f="$1"
+   local crt="/tmp/1.cert"
+   local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+
+   log "Making an ssl certificate"
+
+   openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
+   cat "$crt" >> "$f"
+   rm -f "$crt"
+
+   give_it_to_piler "$f"
+}
+
+
+make_piler_key() {
+   local f="$1"
+
+   log "Generating piler.key"
+
+   dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
+   [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
+
+   give_it_to_piler "$f"
 }
 
 
 fix_configs() {
-   local piler_nginx_conf="/etc/piler/piler-nginx.conf"
+   [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
+   [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
+
+   if [[ ! -f "$PILER_NGINX_CONF" ]]; then
+      log "Writing ${PILER_NGINX_CONF}"
+
+      cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
+      sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
+   fi
 
    if [[ ! -f "$PILER_CONF" ]]; then
-      cp /etc/piler/piler.conf.dist "$PILER_CONF"
+      log "Writing ${PILER_CONF}"
-      chmod 640 "$PILER_CONF"
+
-      chown root:piler "$PILER_CONF"
+      sed \
-      sed -i "s%hostid=.*%hostid=${PILER_HOST%%:*}%" "$PILER_CONF"
+         -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" \
-      sed -i "s%tls_enable=.*%tls_enable=1%" "$PILER_CONF"
+         -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
-      sed -i "s%mysqlpwd=.*%mysqlpwd=${MYSQL_PILER_PASSWORD}%" "$PILER_CONF"
+         -e "s/tls_enable=.*/tls_enable=1/g" \
+         -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
+
+      {
+         echo "mysqlhost=${MYSQL_HOSTNAME}"
+      } >> "$PILER_CONF"
+
+      give_it_to_piler "$PILER_CONF"
    fi
 
-   if [[ ! -f "$piler_nginx_conf" ]]; then
+   if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
-      cp /etc/piler/piler-nginx.conf.dist "$piler_nginx_conf"
+      log "Writing ${CONFIG_SITE_PHP}"
-      sed -i "s%PILER_HOST%${PILER_HOST}%" "$piler_nginx_conf"
+
+      cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
+
+      sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
+
+      {
+         echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
+         echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
+         echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
+         echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
+         echo "\$config['DB_PASSWORD'] = '$MYSQL_PILER_PASSWORD';"
+         echo "\$config['ENABLE_MEMCACHED'] = 1;"
+         echo "\$memcached_server = ['memcached', 11211];"
+      } >> "$CONFIG_SITE_PHP"
    fi
 
-   ln -sf "$piler_nginx_conf" /etc/nginx/sites-enabled/piler
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
-
+       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
-   sed -i "s%HOSTNAME%${PILER_HOST}%" "$CONFIG_SITE_PHP"
+       -e "s%MYSQL_USERNAME%${PILER_USER}%" \
-   sed -i "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" "$CONFIG_SITE_PHP"
+       -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" \
-
+       -i "$SPHINX_CONF"
-   sed -i "s%^\$config\['DECRYPT_BINARY'\].*%\$config\['DECRYPT_BINARY'\] = '/usr/bin/pilerget';%" "$CONFIG_PHP"
-   sed -i "s%^\$config\['DECRYPT_ATTACHMENT_BINARY'\].*%\$config\['DECRYPT_ATTACHMENT_BINARY'\] = '/usr/bin/pileraget';%" "$CONFIG_PHP"
-   sed -i "s%^\$config\['PILER_BINARY'\].*%\$config\['PILER_BINARY'\] = '/usr/sbin/piler';%" "$CONFIG_PHP"
 }
 
 
-service rsyslog start
+wait_until_mysql_server_is_ready() {
-service mysql start
+   while true; do if mysql "--defaults-file=${ROOT_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
 
-create_mysql_db
+   log "${MYSQL_HOSTNAME} is ready"
-pre_seed_sphinx
+}
+
+
+init_database() {
+   local db
+   local has_piler_db=0
+
+   wait_until_mysql_server_is_ready
+
+   while read -r db; do
+      if [[ "$db" == "$MYSQL_DATABASE" ]]; then has_piler_db=1; fi
+   done < <(mysql "--defaults-file=${ROOT_MY_CNF}" <<< 'show databases')
+
+   if [[ $has_piler_db -eq 0 ]]; then
+      log "no ${MYSQL_DATABASE} database, creating"
+
+      mysql "--defaults-file=${ROOT_MY_CNF}" <<< "create database ${MYSQL_DATABASE} character set utf8mb4"
+      mysql "--defaults-file=${ROOT_MY_CNF}" <<< "grant all privileges on ${MYSQL_DATABASE}.* to ${PILER_USER} identified by '${MYSQL_PILER_PASSWORD}'"
+      mysql "--defaults-file=${ROOT_MY_CNF}" <<< "flush privileges"
+
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
+   else
+      log "${MYSQL_DATABASE} db exists"
+   fi
+
+   if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
+   fi
+}
+
+
+create_my_cnf_files() {
+   printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
+      "$MYSQL_HOSTNAME" "$PILER_USER" "$MYSQL_PILER_PASSWORD" "$MYSQL_HOSTNAME" "$PILER_USER" "$MYSQL_PILER_PASSWORD" \
+      > "$PILER_MY_CNF"
+   printf "[client]\nhost = %s\nuser = root\npassword = %s\n" "$MYSQL_HOSTNAME" "$MYSQL_ROOT_PASSWORD" > "$ROOT_MY_CNF"
+
+   give_it_to_piler "$PILER_MY_CNF"
+}
+
+
+start_services() {
+   service rsyslog start
+   service cron start
+   service php7.4-fpm start
+   service nginx start
+}
+
+
+start_piler() {
+   if [[ ! -f "${VOLUME_DIR}/sphinx/main1.spp" ]]; then
+      log "main1.spp does not exist, creating index files"
+      su -c "indexer --all --config ${SPHINX_CONF}" piler
+   fi
+
+   # No pid file should exist for piler
+   rm -f /var/run/piler/*pid
+
+   /etc/init.d/rc.searchd start
+   /etc/init.d/rc.piler start
+}
+
+
+pre_flight_check
 fix_configs
+create_my_cnf_files
+init_database
+start_services
+start_piler
 
-service cron start
+while true; do sleep 3600; done
-service php7.2-fpm start
-service nginx start
-/etc/init.d/rc.searchd start
-
-# fix for overlay, https://github.com/phusion/baseimage-docker/issues/198
-touch /var/spool/cron/crontabs/piler
-
-/etc/init.d/rc.piler start
-
-while true; do sleep 120; done