#!/bin/bash
#
#####
#
# +===================================================+
# |                 © 2019 Privex Inc.                |
# |               https://www.privex.io               |
# +===================================================+
# |                                                   |
# |        File Server Mirror Script                  |
# |                                                   |
# |        Core Developer(s):                         |
# |                                                   |
# |          (+)  Chris (@someguy123) [Privex]        |
# |                                                   |
# +===================================================+
#
#####
# Usage:
#    Adjust REMOTESRV / FILESRV as needed, and configure the folders you want to mirror
#    in MIRROR_FOLDERS / APPEND_FOLDERS

# Master server is se1.files.privex.io
# Slave servers are de1 and fin1.files.privex.io
: ${REMOTESRV="se1.files.privex.io"}
# Base directory for storing file server content
: ${FILESDIR="/filesrv/web"}
# Set to 1 to enable verbose (or just use -v parameter)
: ${FSVERBOSE=0}
# If RESET_OWNER is 1, will run `chmod -R "$NEW_OWNER" folder` after each sync
: ${RESET_OWNER=1}
: ${NEW_OWNER="www-data:www-data"}

# Remote folders to mirror with standard rsync --delete --partial-dir
MIRROR_FOLDERS=(
    "builds"
    "blockchains"
    "images"
    "mirrors"
    "other"
    "pxe"
    "scripts"
)
MIRROR_FILES=(
    "steem/README.txt" "steem/state-README.txt" "hive/state-README.txt" "hive/README.txt"
    #"steemnew/README.txt" "steemnew/state-README.txt"
)


SPARSE_FILES=(
    "blockchains/blurt/shared_memory.bin"
    "blockchains/whaleshares/shared_memory.bin"
    "blockchains/hivetestnet/shared_memory.bin"
)

# Folders containing very large files should be placed in APPEND_FOLDERS
# The folders below will be synced using --append avoiding creation of huge partial files
APPEND_FOLDERS=()
APPEND_FILES=("steem/block_log" "hive/block_log")
#APPEND_FILES=("steem/block_log" "hive/block_log" "steemnew/block_log")

# Folders containing very large files (which aren't always appended to) should be placed in INPLACE_FOLDERS
# The folders below will be synced using --inplace avoiding creation of huge partial files
#INPLACE_FOLDERS=("steem/rocksdb" "hive/rocksdb" "hive/stateshots" "steemnew/rocksdb")
INPLACE_FOLDERS=("steem/rocksdb" "hive/rocksdb" "hive/stateshots")
INPLACE_FILES=(
    "steem/block_log.index" "steem/block_log.lz4"
    #"steemnew/block_log.index" "steemnew/block_log.lz4"
    "hive/block_log.lz4" "hive/block_log.index"
)

####
# Use colors, but only if connected to a terminal, and that terminal
# supports them.
####
if which tput >/dev/null 2>&1; then ncolors=$(tput colors); fi

RED="" GREEN="" YELLOW="" BLUE="" BOLD="" NORMAL="" RESET=""
if [ -t 1 ] && [ -n "$ncolors" ] && [ "$ncolors" -ge 8 ]; then
    RED="$(tput setaf 1)" GREEN="$(tput setaf 2)" YELLOW="$(tput setaf 3)" 
    BLUE="$(tput setaf 4)" BOLD="$(tput bold)" NORMAL="$(tput sgr0)" RESET="$(tput sgr0)"
fi

####
# If you specify the parameter `-v` then that will enable verbose mode
# and change rsync to display full details during the sync
####

if [[ "$#" -gt 0 && "$1" == "-v" ]]; then
    RS_ARGS=('-avh' '--progress') FSVERBOSE=1
else
    RS_ARGS=('-aq')
fi

####
#
# Actual sync code is below
#
####

for mf in "${SPARSE_FILES[@]}"; do
    src="rsync://${REMOTESRV}/${mf}"
    dest="${FILESDIR}/${mf}"
    dest_dir=$(dirname "$dest")
    [[ "$FSVERBOSE" -eq 1 ]] && echo -e "\n---------\n${GREEN}${BOLD}Syncing sparse file $src into $dest with --sparse ... ${RESET}\n---------\n"

    [[ ! -d "$dest_dir" ]] && echo -e "${YELLOW} ---> Creating folder $dest_dir ${RESET} \n" && mkdir -p "$dest_dir"
    rsync "${RS_ARGS[@]}" --sparse "$src" "$dest"
    [[ "$RESET_OWNER" -eq 1 && "$EUID" -eq 0 ]] && chown -R "$NEW_OWNER" "$dest"
done

# For the STEEM blockchain files, use append so we don't have to redownload the whole files

for mf in "${APPEND_FOLDERS[@]}"; do
    src="rsync://${REMOTESRV}/${mf}/"
    dest="${FILESDIR}/${mf}/"
    [[ "$FSVERBOSE" -eq 1 ]] && echo -e "\n---------\n${GREEN}${BOLD}Syncing folder $src into $dest with append ... ${RESET}\n---------\n"

    rsync "${RS_ARGS[@]}" --append "$src" "$dest"
    [[ "$RESET_OWNER" -eq 1 && "$EUID" -eq 0 ]] && chown -R "$NEW_OWNER" "$dest"
done

for mf in "${APPEND_FILES[@]}"; do
    src="rsync://${REMOTESRV}/${mf}"
    dest="${FILESDIR}/${mf}"
    [[ "$FSVERBOSE" -eq 1 ]] && echo -e "\n---------\n${GREEN}${BOLD}Syncing file $src into $dest with append ... ${RESET}\n---------\n"

    rsync "${RS_ARGS[@]}" --append "$src" "$dest"
    [[ "$RESET_OWNER" -eq 1 && "$EUID" -eq 0 ]] && chown -R "$NEW_OWNER" "$dest"
done

for mf in "${INPLACE_FOLDERS[@]}"; do
    src="rsync://${REMOTESRV}/${mf}/"
    dest="${FILESDIR}/${mf}/"
    [[ "$FSVERBOSE" -eq 1 ]] && echo -e "\n---------\n${GREEN}${BOLD}Syncing folder $src into $dest with inplace ... ${RESET}\n---------\n"

    rsync "${RS_ARGS[@]}" --inplace --delete "$src" "$dest"
    [[ "$RESET_OWNER" -eq 1 && "$EUID" -eq 0 ]] && chown -R "$NEW_OWNER" "$dest"
done

for mf in "${INPLACE_FILES[@]}"; do
    src="rsync://${REMOTESRV}/${mf}"
    dest="${FILESDIR}/${mf}"
    [[ "$FSVERBOSE" -eq 1 ]] && echo -e "\n---------\n${GREEN}${BOLD}Syncing file $src into $dest with inplace ... ${RESET}\n---------\n"

    rsync "${RS_ARGS[@]}" --inplace "$src" "$dest"
    [[ "$RESET_OWNER" -eq 1 && "$EUID" -eq 0 ]] && chown -R "$NEW_OWNER" "$dest"
done

# For other mirrored folders, use partial dir to allow automatic resume if something fails to sync
# and delete missing files with --delete
RS_ARGS+=('--partial-dir=".rsync-partial"' '--delete')
# List of remote rsync folders to sync into their respective FILESDIR folder

for mf in "${MIRROR_FOLDERS[@]}"; do
    src="rsync://${REMOTESRV}/${mf}/"
    dest="${FILESDIR}/${mf}/"
    [[ "$FSVERBOSE" -eq 1 ]] && echo -e "\n---------\n${GREEN}${BOLD}Syncing folder $src into $dest ... ${RESET}\n---------\n"

    rsync "${RS_ARGS[@]}" "$src" "$dest"
    [[ "$RESET_OWNER" -eq 1 && "$EUID" -eq 0 ]] && chown -R "$NEW_OWNER" "$dest"
done


for mf in "${MIRROR_FILES[@]}"; do
    src="rsync://${REMOTESRV}/${mf}"
    dest="${FILESDIR}/${mf}"
    [[ "$FSVERBOSE" -eq 1 ]] && echo -e "\n---------\n${GREEN}${BOLD}Syncing file $src into $dest ... ${RESET}\n---------\n"

    rsync "${RS_ARGS[@]}" "$src" "$dest"
    [[ "$RESET_OWNER" -eq 1 && "$EUID" -eq 0 ]] && chown -R "$NEW_OWNER" "$dest"
done