510 lines
18 KiB
Bash
Executable File
510 lines
18 KiB
Bash
Executable File
#!/bin/bash
|
||
# Torrent Mover v8.0 - Enhanced & Robust Version with Directory Deduplication,
|
||
# Improved Archive Handling (keeping archives until ratio limits are reached)
|
||
#
|
||
# This script processes completed torrents reported by Transmission,
|
||
# moving or copying files to designated destination directories.
|
||
# It includes robust locking, advanced error handling & notifications,
|
||
# improved logging, optional post-transfer integrity checks, configurable path mapping,
|
||
# and improved archive extraction that preserves directory structure.
|
||
#
|
||
# Future improvements might include using Transmission’s RPC API.
|
||
|
||
##############################
|
||
# Robust Locking with flock #
|
||
##############################
|
||
LOCK_FILE="/var/lock/torrent-mover.lock"
|
||
exec 200>"${LOCK_FILE}" || { echo "Cannot open lock file" >&2; exit 1; }
|
||
flock -n 200 || { echo "Another instance is running." >&2; exit 1; }
|
||
|
||
##############################
|
||
# Global Runtime Variables #
|
||
##############################
|
||
DRY_RUN=0
|
||
INTERACTIVE=0
|
||
CACHE_WARMUP=0
|
||
DEBUG=0 # Set to 1 if LOG_LEVEL is DEBUG or --debug is passed
|
||
|
||
# To avoid reprocessing the same source directory (across different torrents)
|
||
declare -A processed_source_dirs
|
||
|
||
####################
|
||
# Logging Functions#
|
||
####################
|
||
# All log messages go to stderr.
|
||
log_debug() {
|
||
if [[ "${DEBUG}" -eq 1 ]]; then
|
||
echo -e "[DEBUG] $(date '+%F %T') - $*" | tee -a "${LOG_FILE}" >&2
|
||
[[ "${USE_SYSLOG}" == "true" ]] && logger -t torrent-mover "[DEBUG] $*"
|
||
fi
|
||
}
|
||
log_info() {
|
||
echo -e "[INFO] $(date '+%F %T') - $*" | tee -a "${LOG_FILE}" >&2
|
||
[[ "${USE_SYSLOG}" == "true" ]] && logger -t torrent-mover "[INFO] $*"
|
||
}
|
||
log_warn() {
|
||
echo -e "[WARN] $(date '+%F %T') - $*" | tee -a "${LOG_FILE}" >&2
|
||
[[ "${USE_SYSLOG}" == "true" ]] && logger -t torrent-mover "[WARN] $*"
|
||
}
|
||
log_error() {
|
||
echo -e "[ERROR] $(date '+%F %T') - $*" | tee -a "${LOG_FILE}" >&2
|
||
[[ "${USE_SYSLOG}" == "true" ]] && logger -t torrent-mover "[ERROR] $*"
|
||
}
|
||
|
||
#################################
|
||
# Error Handling & Notifications#
|
||
#################################
|
||
error_handler() {
|
||
local lineno="$1"
|
||
local msg="$2"
|
||
log_error "Error on line ${lineno}: ${msg}"
|
||
# Optionally send a notification (e.g., email)
|
||
exit 1
|
||
}
|
||
trap 'error_handler ${LINENO} "$BASH_COMMAND"' ERR
|
||
|
||
##############################
|
||
# Configuration & Validation #
|
||
##############################
|
||
CONFIG_FILE="/etc/torrent/mover.conf"
|
||
if [[ ! -f "${CONFIG_FILE}" ]]; then
|
||
echo "FATAL: Configuration file missing: ${CONFIG_FILE}" >&2
|
||
exit 1
|
||
fi
|
||
source "${CONFIG_FILE}"
|
||
|
||
# Validate required configuration values.
|
||
if [[ -z "${TRANSMISSION_PATH_PREFIX:-}" || -z "${LOCAL_PATH_PREFIX:-}" ]]; then
|
||
echo "FATAL: TRANSMISSION_PATH_PREFIX and LOCAL_PATH_PREFIX must be set in ${CONFIG_FILE}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ "${LOG_LEVEL}" == "DEBUG" ]]; then
|
||
DEBUG=1
|
||
fi
|
||
|
||
# Parse STORAGE_DIRS into an array.
|
||
STORAGE_DIRS_ARRAY=()
|
||
if [[ -n "${STORAGE_DIRS}" ]]; then
|
||
IFS=',' read -ra STORAGE_DIRS_ARRAY <<< "${STORAGE_DIRS}"
|
||
fi
|
||
|
||
##############################
|
||
# Helper & Utility Functions #
|
||
##############################
|
||
|
||
# translate_source: Converts the Transmission‑reported path into the local path.
|
||
translate_source() {
|
||
local src="$1"
|
||
echo "${src/#${TRANSMISSION_PATH_PREFIX}/${LOCAL_PATH_PREFIX}}"
|
||
}
|
||
|
||
# parse_args: Processes command‑line options.
|
||
parse_args() {
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--dry-run) DRY_RUN=1; shift ;;
|
||
--interactive) INTERACTIVE=1; shift ;;
|
||
--cache-warmup) CACHE_WARMUP=1; shift ;;
|
||
--debug) DEBUG=1; shift ;;
|
||
--help)
|
||
echo "Usage: $0 [--dry-run] [--interactive] [--cache-warmup] [--debug]" >&2
|
||
exit 0
|
||
;;
|
||
*) echo "Invalid option: $1" >&2; exit 1 ;;
|
||
esac
|
||
done
|
||
}
|
||
|
||
# check_dependencies: Ensures required commands are available.
|
||
check_dependencies() {
|
||
local deps=("transmission-remote" "unrar" "unzip" "7z" "parallel" "bc")
|
||
for dep in "${deps[@]}"; do
|
||
command -v "${dep}" >/dev/null 2>&1 || { log_error "Missing dependency: ${dep}"; exit 1; }
|
||
done
|
||
}
|
||
|
||
# check_disk_usage: Warn if disk usage is over 90%.
|
||
declare -A CHECKED_MOUNTS=()
|
||
check_disk_usage() {
|
||
local dir="$1"
|
||
[[ -z "${dir}" ]] && return
|
||
if ! df -P "${dir}" &>/dev/null; then
|
||
log_warn "Directory not found: ${dir}"
|
||
return
|
||
fi
|
||
local mount_point
|
||
mount_point=$(df -P "${dir}" | awk 'NR==2 {print $6}')
|
||
[[ -z "${mount_point}" ]] && return
|
||
if [[ -z "${CHECKED_MOUNTS["${mount_point}"]+x}" ]]; then
|
||
local usage
|
||
usage=$(df -P "${dir}" | awk 'NR==2 {sub(/%/, "", $5); print $5}')
|
||
if (( usage >= 90 )); then
|
||
log_warn "Storage warning: ${mount_point} at ${usage}% capacity"
|
||
fi
|
||
CHECKED_MOUNTS["${mount_point}"]=1
|
||
fi
|
||
}
|
||
|
||
# init_checksum_db: Initializes the checksum database.
|
||
init_checksum_db() {
|
||
mkdir -p "$(dirname "${CHECKSUM_DB}")"
|
||
touch "${CHECKSUM_DB}" || { log_error "Could not create ${CHECKSUM_DB}"; exit 1; }
|
||
chmod 600 "${CHECKSUM_DB}"
|
||
}
|
||
|
||
# record_checksums: Generates checksums for files in given directories.
|
||
record_checksums() {
|
||
log_info "Generating checksums with ${PARALLEL_THREADS:-$(nproc)} threads"
|
||
find "$@" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -print0 | \
|
||
parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | sort > "${CHECKSUM_DB}.tmp"
|
||
mv "${CHECKSUM_DB}.tmp" "${CHECKSUM_DB}"
|
||
}
|
||
|
||
# file_metadata: Returns an md5 hash for file metadata.
|
||
file_metadata() {
|
||
find "$1" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort | awk '{print $1}'
|
||
}
|
||
|
||
# files_need_processing: Checks if the source files need processing.
|
||
files_need_processing() {
|
||
local src="$1"
|
||
shift
|
||
local targets=("$@")
|
||
|
||
if [[ ! -d "${src}" ]]; then
|
||
log_warn "Source directory missing: ${src}"
|
||
return 1
|
||
fi
|
||
|
||
log_info "=== FILE VERIFICATION DEBUG START ==="
|
||
log_info "Source directory: ${src}"
|
||
log_info "Verification targets: ${targets[*]}"
|
||
|
||
local empty_target_found=0
|
||
for target in "${targets[@]}"; do
|
||
if [[ ! -d "${target}" ]]; then
|
||
log_info "Target missing: ${target}"
|
||
empty_target_found=1
|
||
continue
|
||
fi
|
||
|
||
local file_count
|
||
file_count=$(find "${target}" -mindepth 1 -maxdepth 1 -print | wc -l)
|
||
log_debug "File count for target ${target}: ${file_count}"
|
||
if [[ "${file_count}" -eq 0 ]]; then
|
||
log_info "Empty target directory: ${target}"
|
||
empty_target_found=1
|
||
else
|
||
log_info "Target contains ${file_count} items: ${target}"
|
||
log_info "First 5 items:"
|
||
find "${target}" -mindepth 1 -maxdepth 1 | head -n 5 | while read -r item; do
|
||
log_info " - ${item##*/}"
|
||
done
|
||
fi
|
||
done
|
||
|
||
if [[ "${empty_target_found}" -eq 1 ]]; then
|
||
log_info "Empty target detected - processing needed"
|
||
log_info "=== FILE VERIFICATION DEBUG END ==="
|
||
return 0
|
||
fi
|
||
|
||
log_info "Generating source checksums..."
|
||
local src_checksums
|
||
src_checksums=$(find "${src}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
|
||
log_info "First 5 source checksums:"
|
||
echo "${src_checksums}" | head -n 5 | while read -r line; do
|
||
log_info " ${line}"
|
||
done
|
||
|
||
local match_found=0
|
||
for target in "${targets[@]}"; do
|
||
log_info "Checking against target: ${target}"
|
||
log_info "Generating target checksums..."
|
||
local target_checksums
|
||
target_checksums=$(find "${target}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
|
||
log_info "First 5 target checksums:"
|
||
echo "${target_checksums}" | head -n 5 | while read -r line; do
|
||
log_info " ${line}"
|
||
done
|
||
|
||
if diff <(echo "${src_checksums}") <(echo "${target_checksums}") >/dev/null; then
|
||
log_info "Exact checksum match found in: ${target}"
|
||
match_found=1
|
||
break
|
||
else
|
||
log_info "No match in: ${target}"
|
||
fi
|
||
done
|
||
|
||
log_info "=== FILE VERIFICATION DEBUG END ==="
|
||
[[ "${match_found}" -eq 1 ]] && return 1 || return 0
|
||
}
|
||
|
||
# warm_cache: Pre-calculates checksums for storage directories.
|
||
warm_cache() {
|
||
log_info "Starting cache warmup for Movies..."
|
||
local targets=("${DIR_MOVIES_DST}" "${STORAGE_DIRS_ARRAY[@]}")
|
||
record_checksums "${targets[@]}"
|
||
log_info "Cache warmup completed. Checksums stored in ${CHECKSUM_DB}"
|
||
}
|
||
|
||
# is_processed: Checks if the torrent (by hash) has already been processed.
|
||
is_processed() {
|
||
grep -q "^${1}$" "${PROCESSED_LOG}" 2>/dev/null
|
||
}
|
||
|
||
# mark_processed: Records a processed torrent.
|
||
mark_processed() {
|
||
echo "${1}" >> "${PROCESSED_LOG}"
|
||
}
|
||
|
||
# get_destination: Maps a source directory to a destination directory based on keywords.
|
||
declare -A PATH_CACHE
|
||
get_destination() {
|
||
local source_path="$1"
|
||
if [[ -n "${PATH_CACHE["${source_path}"]+x}" ]]; then
|
||
echo "${PATH_CACHE["${source_path}"]}"
|
||
return
|
||
fi
|
||
log_info "Analyzing path: ${source_path}"
|
||
local destination
|
||
case "${source_path,,}" in
|
||
*games*) destination="${DIR_GAMES_DST}";;
|
||
*apps*) destination="${DIR_APPS_DST}";;
|
||
*movies*) destination="${DIR_MOVIES_DST}";;
|
||
*books*) destination="${DIR_BOOKS_DST}";;
|
||
*) destination="${DEFAULT_DST}";;
|
||
esac
|
||
log_info "Mapped to: ${destination}"
|
||
PATH_CACHE["${source_path}"]="${destination}"
|
||
echo "${destination}"
|
||
}
|
||
|
||
######################################
|
||
# Improved Archive Extraction Handler #
|
||
######################################
|
||
# For each archive found in the source directory, create a subdirectory in the destination
|
||
# named after the archive (without its extension) and extract into that subdirectory.
|
||
# IMPORTANT: The archive is now retained in the source, so it will remain until the ratio
|
||
# limits are reached and Transmission removes the torrent data.
|
||
handle_archives() {
|
||
local src="$1" dst="$2"
|
||
find "${src}" -type f \( -iname "*.rar" -o -iname "*.zip" -o -iname "*.7z" \) | while read -r arch; do
|
||
log_info "Extracting archive: ${arch}"
|
||
local base
|
||
base=$(basename "${arch}")
|
||
local subdir="${dst}/${base%.*}"
|
||
mkdir -p "${subdir}" || { log_error "Failed to create subdirectory ${subdir}"; continue; }
|
||
case "${arch##*.}" in
|
||
rar)
|
||
unrar x -o- "${arch}" "${subdir}" || { log_error "unrar failed for ${arch}"; continue; }
|
||
;;
|
||
zip)
|
||
unzip -o "${arch}" -d "${subdir}" || { log_error "unzip failed for ${arch}"; continue; }
|
||
;;
|
||
7z)
|
||
7z x "${arch}" -o"${subdir}" || { log_error "7z extraction failed for ${arch}"; continue; }
|
||
;;
|
||
esac
|
||
log_info "Archive ${arch} retained in source until ratio limits are reached."
|
||
done
|
||
}
|
||
|
||
# move_files: Moves files using parallel processing if enabled.
|
||
move_files() {
|
||
if (( PARALLEL_PROCESSING )); then
|
||
parallel -j ${PARALLEL_THREADS:-$(nproc)} mv {} "${1}" ::: "${2}"/*
|
||
else
|
||
mv "${2}"/* "${1}"
|
||
fi
|
||
}
|
||
|
||
# copy_files: Copies files using parallel processing if enabled.
|
||
copy_files() {
|
||
if (( PARALLEL_PROCESSING )); then
|
||
parallel -j ${PARALLEL_THREADS:-$(nproc)} cp -r {} "${1}" ::: "${2}"/*
|
||
else
|
||
cp -r "${2}"/* "${1}"
|
||
fi
|
||
}
|
||
|
||
# process_copy: Validates directories, then copies/moves files from source to destination.
|
||
# Optionally verifies integrity after transfer if CHECK_TRANSFER_INTEGRITY is "true".
|
||
process_copy() {
|
||
local id="$1" hash="$2" src="$3" dst="$4"
|
||
if [[ ! -d "${src}" ]]; then
|
||
log_error "Source directory missing: ${src}"
|
||
return 1
|
||
fi
|
||
if [[ ! -d "${dst}" ]]; then
|
||
log_info "Creating destination directory: ${dst}"
|
||
mkdir -p "${dst}" || { log_error "Failed to create directory: ${dst}"; return 1; }
|
||
chmod 775 "${dst}"
|
||
chown debian-transmission:debian-transmission "${dst}"
|
||
fi
|
||
if [[ ! -w "${dst}" ]]; then
|
||
log_error "No write permissions for: ${dst}"
|
||
return 1
|
||
fi
|
||
if (( DRY_RUN )); then
|
||
log_info "[DRY RUN] Would process torrent ${id}:"
|
||
log_info " - Copy files from ${src} to ${dst}"
|
||
log_info " - File count: $(find "${src}" -maxdepth 1 -type f | wc -l)"
|
||
return
|
||
fi
|
||
handle_archives "${src}" "${dst}"
|
||
case "${COPY_MODE}" in
|
||
move)
|
||
log_info "Moving files from ${src} to ${dst}"
|
||
move_files "${dst}" "${src}"
|
||
;;
|
||
copy)
|
||
log_info "Copying files from ${src} to ${dst}"
|
||
copy_files "${dst}" "${src}"
|
||
;;
|
||
esac
|
||
if [ $? -eq 0 ]; then
|
||
if [[ "${CHECK_TRANSFER_INTEGRITY}" == "true" ]]; then
|
||
log_info "Verifying integrity of transferred files..."
|
||
local src_checksum target_checksum
|
||
src_checksum=$(find "${src}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
|
||
target_checksum=$(find "${dst}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
|
||
if diff <(echo "${src_checksum}") <(echo "${target_checksum}") >/dev/null; then
|
||
log_info "Integrity check passed."
|
||
else
|
||
log_error "Integrity check FAILED for ${src}"
|
||
return 1
|
||
fi
|
||
fi
|
||
log_info "Transfer completed successfully"
|
||
mark_processed "${hash}"
|
||
else
|
||
log_error "Transfer failed for ${src}"
|
||
fi
|
||
}
|
||
|
||
# process_removal: Removes a torrent via Transmission.
|
||
process_removal() {
|
||
local id="$1"
|
||
if (( DRY_RUN )); then
|
||
log_info "[DRY RUN] Would remove torrent ${id}"
|
||
return
|
||
fi
|
||
transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
|
||
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" \
|
||
-t "${id}" --remove-and-delete
|
||
}
|
||
|
||
#################
|
||
# Main Function #
|
||
#################
|
||
main() {
|
||
check_dependencies
|
||
|
||
# Validate destination directories.
|
||
declare -a REQUIRED_DIRS=(
|
||
"${DIR_GAMES_DST}"
|
||
"${DIR_APPS_DST}"
|
||
"${DIR_MOVIES_DST}"
|
||
"${DIR_BOOKS_DST}"
|
||
"${DEFAULT_DST}"
|
||
)
|
||
for dir in "${REQUIRED_DIRS[@]}"; do
|
||
if [[ ! -d "${dir}" ]]; then
|
||
log_error "Directory missing: ${dir}"
|
||
exit 1
|
||
fi
|
||
if [[ ! -w "${dir}" ]]; then
|
||
log_error "Write permission denied: ${dir}"
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
init_checksum_db
|
||
|
||
if (( CACHE_WARMUP )); then
|
||
warm_cache
|
||
exit 0
|
||
fi
|
||
|
||
log_info "Starting processing"
|
||
declare -A warned_dirs=()
|
||
transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
|
||
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -l | awk 'NR>1 && $1 ~ /^[0-9]+$/ {print $1}' | while read -r id; do
|
||
|
||
local info
|
||
info=$(transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
|
||
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -t "${id}" -i)
|
||
local hash
|
||
hash=$(grep "Hash:" <<< "${info}" | awk '{print $2}')
|
||
local ratio
|
||
ratio=$(grep "Ratio:" <<< "${info}" | awk '{print $2 == "None" ? 0 : $2}' | tr -cd '0-9.')
|
||
ratio=${ratio:-0}
|
||
local time
|
||
time=$(grep "Seeding Time:" <<< "${info}" | awk '{print $3 == "None" ? 0 : $3}' | tr -cd '0-9.')
|
||
time=${time:-0}
|
||
local percent_done
|
||
percent_done=$(grep "Percent Done:" <<< "${info}" | awk '{gsub(/%/, ""); print $3 == "None" ? 0 : $3}')
|
||
percent_done=${percent_done:-0}
|
||
|
||
# Extract Transmission-reported directory and translate to local path.
|
||
local reported_dir
|
||
reported_dir=$(grep -i "Location:" <<< "${info}" | awk -F": " '{print $2}' | xargs)
|
||
local dir
|
||
dir=$(translate_source "${reported_dir}")
|
||
log_info "Torrent source directory reported: '${reported_dir}' translated to '${dir}'"
|
||
local dst
|
||
dst=$(get_destination "${dir}")
|
||
[[ -z "${warned_dirs["${dir}"]+x}" ]] && warned_dirs["${dir}"]=0
|
||
|
||
# Avoid processing the same directory more than once.
|
||
if [[ -n "${processed_source_dirs["${dir}"]+x}" ]]; then
|
||
log_info "Directory ${dir} has already been processed; skipping copy for torrent ${id}"
|
||
elif (( $(bc <<< "${percent_done} >= 100") )) && ! is_processed "${hash}"; then
|
||
log_info "Processing completed torrent ${id} (${percent_done}% done)"
|
||
if [[ "${dst}" == "${DEFAULT_DST}" ]] && (( warned_dirs["${dir}"] == 0 )); then
|
||
log_warn "Using default destination for: ${dir}"
|
||
warned_dirs["${dir}"]=1
|
||
fi
|
||
local targets=("${dst}")
|
||
case "${dst}" in
|
||
"${DIR_MOVIES_DST}") targets+=("${STORAGE_DIRS_ARRAY[@]}");;
|
||
esac
|
||
if ! files_need_processing "${dir}" "${targets[@]}"; then
|
||
log_info "Skipping copy - files already exist in:"
|
||
for target in "${targets[@]}"; do
|
||
[[ -d "${target}" ]] && log_info " - ${target}"
|
||
done
|
||
else
|
||
process_copy "${id}" "${hash}" "${dir}" "${dst}"
|
||
processed_source_dirs["${dir}"]=1
|
||
fi
|
||
fi
|
||
|
||
if (( $(bc <<< "${ratio} >= ${SEED_RATIO}") )) || (( $(bc <<< "${time} >= ${SEED_TIME}") )); then
|
||
log_info "Removing torrent ${id} (Ratio: ${ratio}, Time: ${time})"
|
||
process_removal "${id}"
|
||
fi
|
||
done
|
||
|
||
check_disk_usage "${DIR_GAMES_DST}"
|
||
check_disk_usage "${DIR_APPS_DST}"
|
||
check_disk_usage "${DIR_MOVIES_DST}"
|
||
check_disk_usage "${DIR_BOOKS_DST}"
|
||
check_disk_usage "${DEFAULT_DST}"
|
||
}
|
||
|
||
######################
|
||
# Command-line Parsing #
|
||
######################
|
||
parse_args "$@"
|
||
|
||
if (( INTERACTIVE )); then
|
||
read -rp "Confirm processing? (y/n) " choice
|
||
[[ "${choice}" =~ ^[Yy]$ ]] || exit 0
|
||
fi
|
||
|
||
main
|