#!/bin/bash
# (C) 2020 Zefie Networks
# This script scans for .rar files in a directory, and runs a test on them using the unrar command
# This script requires the nonfree version of unrar and is only tested with v5
# The results of the scan will be saved in a file (defined by Z_TESTDAT below)
# The file is designed to be both human readable and readable by this script, so do not modify
# the output file except to delete a line to force revalidation (DOS/Unix linefeeds shouldnt matter).
# This script will stop if it reaches a damaged RAR if Z_ABORT_ON_ERROR is defined.
# This script will only validate files that have either not been checked yet, modified since the last check,
# or it has been Z_EXPIRE_TIME since the last validation.
# CONFIG
# How long since the last successful validation before we validate the file again?
Z_EXPIRE_TIME="250:0:0:0" # days:hours:minutes:seconds
# Filename for test metadata, will be placed in directory specified to scan
# Layout is tab-delimited, as follows
# *** Date Validated Test Status Recovery Record Status RAR Size RAW Size Filename
Z_TESTDAT_FILENAME="rar_test_status.txt"
# You can change these if you want but be aware of tabbing and cosmetic issues
Z_OKSTR="PASSED (OK)"
Z_RECOVERY_RECORD_OKSTR="Has Recovery Record"
Z_RECOVERY_RECORD_ERRSTR="NO RECOVERY RECORD"
Z_DATESTR="%Y-%m-%d %H:%M:%S"
Z_DATESTR_DAY="%A, %B %d %Y"
# Comment this out to continue scanning even if a rar fails to validate
Z_ABORT_ON_ERROR=1
# Uncomment this out to fail if a rar does not have a recovery record
#Z_ABORT_ON_NRR=1
# END EZ CONFIG
Z_SCRIPTDIR="$(realpath "$(dirname "${0}")")"
Z_ERRCNT=0
Z_ERRRCNT=0
Z_NRRCNT=0
Z_MISSING=0
# shellcheck source=/home/zefie/bin/ansihelper.sh
source "${Z_SCRIPTDIR}/ansihelper.sh"
if [ -z "${1}" ]; then
# script requires an arg to run
echo "Usage: ${0} <directory> [directory ...]"
exit 1;
fi
if [ "${1}" == "-nf" ]; then
unset Z_ABORT_ON_ERROR;
shift;
fi
for arg in "${@}"; do
if [ ! -d "${arg}" ]; then
echo "ERR: Not a directory: \"${1}\""
fi
done
# Intercept ^C and finish up cleanly instead of getting stuck in the loop
trap ctrl_c INT
function ctrl_c() {
echo "** ^C pressed, aborting script, please wait..."
export Z_ABORT=1;
}
function z_chk_if_single_or_part1_rar() {
unset Z_MULTIPART_RAR
# Check if a single part rar, or if part1 of a multipart, returns 1 if not.
if [ "$(echo "${1}" | grep -Eic '\.part[0-9]{0,3}\.rar$')" -gt 0 ]; then
if [ "$(echo "${1}" | grep -Eic '\.part[0]{0,2}1\.rar$')" -eq 0 ]; then
return 1;
fi
export Z_MULTIPART_RAR=1
fi
}
function bytesToHuman() {
# Simple 'friendlyBytes' from stackoverflow
b=${1:-0}; d=''; s=0; S=(Bytes {K,M,G,T,P,E,Z,Y}iB)
while ((b > 1024)); do
d="$(printf ".%02d" $((b % 1024 * 100 / 1024)))"
b=$((b / 1024))
(( s++ ))
done
echo "$b$d ${S[$s]}"
}
function z_strip_empty_array() {
# Usage: z_strip_empty_array $arrayname $thearray
# (Re)defines the array of $arrayname with the contents of $thearray, but without null entries (and re-sorted)
local Z_ARRY="${1}"
shift
eval "declare -a ${Z_ARRY}"
for i in "${@}"; do
if [ -z "$i" ]; then
continue
fi
eval "${Z_ARRY}+=(\"${i}\")"
done
}
function z_strip_from_txt() {
# Rebuilds text file $2 without the lines containing $1
local Z_TMP
Z_TMP="$(mktemp)"
grep -Fv "${1}" "${2}" > "${Z_TMP}"
cat "${Z_TMP}" > "${2}"
rm "${Z_TMP}"
}
function z_fileheader() {
# Rebuilds the text header (starting with ***) without destroying the file contents
# No args, replace $Z_TESTDAT with $1 to make into a arg type function
# You can add extra tabs for formatting (eg \t\t) without breaking the script, thanks to z_strip_empty_array
local Z_TMP Z_EXPIRE
Z_TMP="$(mktemp)"
Z_EXPIRE=$(("$(date -u +%s)" + "$(echo -n "${Z_EXPIRE_TIME}" | awk -F: '{ print ($1 * 86400) + ($2 * 3600) + ($3 * 60) + $4 }')"))
if [ -f "${1}" ]; then
grep -Fv "***" "${1}" > "${Z_TMP}"
fi
echo "*** RAR Integrty Test Status for ${Z_DIR} (Times in UTC/GMT) ***" > "${1}"
{
echo "*** Last Updated: $(date -u "+${Z_DATESTR}") (Suggested Revalidation: $(date -u --date="@${Z_EXPIRE}" "+${Z_DATESTR_DAY}")) ***"
echo -e '*** Date Validated\tTest Status\tRecovery Record Status\tRAR Size\tRAW Size\tFile Last Modified\tFilename'
cat "${Z_TMP}"
} >> "${1}"
rm "${Z_TMP}"
}
function z_chk_abort() {
# If the abort flag has been set prior, exit now
if [ ! -z "${Z_ABORT}" ]; then
exit "${Z_ABORT}";
fi
}
function do_rar_test() {
z_chk_abort
local Z_STATUS="UNKNOWN ERR"
local Z_STATUS_C="yellow"
local Z_REC_C="yellow"
local Z_RRES Z_TMP Z_REC Z_COMPSIZE Z_RAWSIZE Z_LASTMOD Z_REC_REC Z_REC_RES;
# Check for recovery record
Z_REC_REC="$(unrar l "${1}" | grep "recovery record" -c)"
Z_REC_RES=$?
if [ ${Z_REC_RES} -ne 0 ] || [ "${Z_REC_REC}" -lt 1 ]; then
Z_REC="${Z_RECOVERY_RECORD_ERRSTR}"
Z_NRRCNT=$((Z_ERRCNT + 1))
if [ ! -z "${Z_ABORT_ON_NRR}" ]; then
Z_ABORT=1
fi
Z_REC_C="red"
else
Z_REC="${Z_RECOVERY_RECORD_OKSTR}"
Z_REC_C="green"
fi
# Test with unrar (7z/7za/p7zip will return errcode 0 on corrupt file and cannot be used)
unrar t "${1}"
Z_RRES=$?
if [ ${Z_RRES} -eq 0 ]; then
Z_STATUS="${Z_OKSTR}"
Z_STATUS_C="green"
else
case ${Z_RRES} in
255)
# When user hits ^C
Z_STATUS="FAILED (ABORT)"
Z_ABORT=1
;;
10)
Z_STATUS="FAILED (NOTRAR)"
;;
3)
if [ "${Z_REC}" == "${Z_RECOVERY_RECORD_OKSTR}" ]; then
# corrupt and has recovery record
Z_STATUS="NEEDS REPAIR"
Z_ERRRCNT=$((Z_ERRRCNT))
else
# corrupt and does NOT have recovery record
Z_STATUS="FAILED (CORRUPT)"
fi
;;
*)
# catchall for unknown error codes
Z_STATUS="FAILED (${Z_RRES})"
;;
esac
if [ ! -z "${Z_ABORT_ON_ERROR}" ]; then
Z_ABORT=1
fi
Z_STATUS_C="red"
Z_ERRCNT=$((Z_ERRCNT + 1))
fi
z_strip_from_txt "${1}" "${2}"
z_fileheader "${2}"
if [ -z "${Z_MULTIPART_RAR}" ]; then
# Use stat to get filesize of a single file
Z_COMPSIZE="$(bytesToHuman "$(stat -c %s "${1}")")"
else
# Use stat to get the size of all the rar parts, and add them together
Z_MULTIRAR_PREFIX="$(echo "${1}" | rev | cut -d'.' -f3- | rev).part"
Z_COMPSIZE=0
for fss in $(stat -c %s "${Z_MULTIRAR_PREFIX}"*); do
Z_COMPSIZE=$(( Z_COMPSIZE + fss ))
done
Z_COMPSIZE=$(bytesToHuman ${Z_COMPSIZE})
fi
Z_RAWSIZE="$(bytesToHuman "$(unrar l "${1}" | tail -n2 | head -n1 | awk ' { print $1 } ')")"
Z_LASTMOD="$(date -u "+${Z_DATESTR}" --date="@$(stat -c "%Y" "${1}")")"
echo -e "$(date -u "+${Z_DATESTR}")\\t${Z_STATUS}\\t${Z_REC}\\t${Z_COMPSIZE}\\t${Z_RAWSIZE}\\t${Z_LASTMOD}\\t${1}" >> "${2}"
echo -e "$(str_color "${Z_STATUS_C}" "${Z_STATUS}")\\t$(str_color "${Z_REC_C}" "${Z_REC}")\\t${1}"
}
function z_process_directory() {
local Z_DIR Z_TESTDAT Z_TS Z_EXPIRE Z_LASTMOD Z_RES Z_PWD Z_MKTEMP;
Z_DIR="$(realpath "${1}")"
Z_PWD="${PWD}"
Z_TESTDAT="${Z_DIR}/${Z_TESTDAT_FILENAME}"
cd "${Z_DIR}" || (echo "Could not enter directory: ${Z_DIR}" && exit 1)
while IFS=$'\n' read -r -d '' file
do
z_chk_abort
z_chk_if_single_or_part1_rar "${file}" || continue;
# if the test result file doesn't exist
if [ ! -f "${Z_TESTDAT}" ]; then
# then create it and insert the header
touch "${Z_TESTDAT}" || exit 1;
z_fileheader "${Z_TESTDAT}"
fi
# scan the test result file for this filename
IFS=$'\t' read -r -a Z_RES <<< "$(grep -F "${file}" "${Z_TESTDAT}")"
z_strip_empty_array Z_RES "${Z_RES[@]}";
# if there was a match from the test result file
if [ ${#Z_RES[@]} -gt 2 ]; then
# get the stored information and ch eck it
Z_TS="$(date -u --date="${Z_RES[0]}" +%s 2>/dev/null || echo -n "${Z_RES[0]}")"
Z_EXPIRE=$(("${Z_TS}" + "$(echo -n "${Z_EXPIRE_TIME}" | awk -F: '{ print ($1 * 86400) + ($2 * 3600) + ($3 * 60) + $4 }')"))
Z_LASTMOD="$(date -u "+${Z_DATESTR}" --date="@$(stat -c "%Y" "${file}")")"
# if the expire time elapsed, the last test wasn't ok, or the modified time changed
if [ "$(date -u +%s)" -gt "${Z_EXPIRE}" ] || [ "${Z_RES[1]}" != "${Z_OKSTR}" ] || [ "${Z_LASTMOD}" != "${Z_RES[5]}" ]; then
# then validate the file
do_rar_test "${file}" "${Z_TESTDAT}"
else
# scan status file for any rars that don't have recovery records and count them
if [ "${Z_RES[2]}" != "${Z_RECOVERY_RECORD_OKSTR}" ]; then
Z_NRRCNT=$((Z_NRRCNT + 1))
fi
fi
else
# otherwise validate the file
do_rar_test "${file}" "${Z_TESTDAT}"
fi
done < <(find . -name '*.rar' -print0);
Z_MKTEMP=$(mktemp)
while IFS=$'\n' read -r line
do
z_chk_abort
if [ "$(echo "${line}" | grep -F "***" -c)" -gt 0 ]; then
echo "${line}" >> "${Z_MKTEMP}"
continue;
fi
IFS=$'\t' read -r -a Z_TRES <<< "${line}"
if [ ! -z "${Z_TRES[6]}" ]; then
if [ ! -f "${Z_TRES[6]}" ]; then
Z_MISSING=$((Z_MISSING + 1))
else
echo "${line}" >> "${Z_MKTEMP}"
fi
fi
done < "${Z_TESTDAT}";
if [ "${Z_MISSING}" -gt 0 ]; then
cat "${Z_MKTEMP}" > "${Z_TESTDAT}"
fi
rm "${Z_MKTEMP}"
cd "${Z_PWD}" || (echo "Could not return to original directory: ${Z_PWD}" && exit 1)
if [ "${Z_ERRCNT}" -eq 0 ] && [ "${Z_NRRCNT}" -eq 0 ]; then
# all good
echo "All $(str_color green "${Z_OKSTR}") for ${Z_DIR}"
else
echo "$(str_color red "Error report") for ${Z_DIR}:"
local Z_ERRRCNT_STR;
if [ "${Z_ERRRCNT}" -eq "${Z_ERRCNT}" ]; then
Z_ERRRCNT_STR="$(str_color green all)";
elif [ "${Z_ERRRCNT}" -gt 0 ]; then
Z_ERRRCNT_STR="$(str_color yellow "${Z_ERRRCNT}")"
else
Z_ERRRCNT_STR="$(str_color red none)"
fi
if [ "${Z_ERRCNT}" -gt 0 ]; then
echo "$(str_color red "${Z_ERRCNT} file(s) failed") to validate (of those, ${Z_ERRRCNT_STR} of them can be repaired)."
fi
if [ "${Z_NRRCNT}" -gt 0 ]; then
echo "$(str_color yellow "${Z_NRRCNT} file(s)") do not have recovery records."
fi
fi
if [ "${Z_MISSING}" -gt 0 ]; then
echo "$(str_color yellow "Removed ${Z_MISSING} missing file(s)") from $(str_color cyan "${Z_TESTDAT}")"
fi
}
for dir in "${@}"; do
z_chk_abort
z_process_directory "${dir}"
done
Comments