some program redesign
This commit is contained in:
masterdraco
+52
-380
@@ -1,14 +1,16 @@
|
||||
#!/bin/bash
|
||||
# Torrent Mover v8.0 - Enhanced & Robust Version with Directory Deduplication,
|
||||
# Improved Archive Handling (keeping archives until ratio limits are reached)
|
||||
# Torrent Mover v8.0 - Enhanced & Robust Version with modular architecture,
|
||||
# improved error handling, security, and content categorization
|
||||
#
|
||||
# This script processes completed torrents reported by Transmission,
|
||||
# moving or copying files to designated destination directories.
|
||||
# It includes robust locking, advanced error handling & notifications,
|
||||
# improved logging, optional post-transfer integrity checks, configurable path mapping,
|
||||
# and improved archive extraction that preserves directory structure.
|
||||
#
|
||||
# Future improvements might include using Transmission’s RPC API.
|
||||
|
||||
# Set script location for importing modules
|
||||
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
||||
LIB_DIR="/usr/local/lib/torrent-mover"
|
||||
|
||||
##############################
|
||||
# Robust Locking with flock #
|
||||
@@ -17,52 +19,6 @@ LOCK_FILE="/var/lock/torrent-mover.lock"
|
||||
exec 200>"${LOCK_FILE}" || { echo "Cannot open lock file" >&2; exit 1; }
|
||||
flock -n 200 || { echo "Another instance is running." >&2; exit 1; }
|
||||
|
||||
##############################
|
||||
# Global Runtime Variables #
|
||||
##############################
|
||||
DRY_RUN=0
|
||||
INTERACTIVE=0
|
||||
CACHE_WARMUP=0
|
||||
DEBUG=0 # Set to 1 if LOG_LEVEL is DEBUG or --debug is passed
|
||||
|
||||
# To avoid reprocessing the same source directory (across different torrents)
|
||||
declare -A processed_source_dirs
|
||||
|
||||
####################
|
||||
# Logging Functions#
|
||||
####################
|
||||
# All log messages go to stderr.
|
||||
log_debug() {
|
||||
if [[ "${DEBUG}" -eq 1 ]]; then
|
||||
echo -e "[DEBUG] $(date '+%F %T') - $*" | tee -a "${LOG_FILE}" >&2
|
||||
[[ "${USE_SYSLOG}" == "true" ]] && logger -t torrent-mover "[DEBUG] $*"
|
||||
fi
|
||||
}
|
||||
log_info() {
|
||||
echo -e "[INFO] $(date '+%F %T') - $*" | tee -a "${LOG_FILE}" >&2
|
||||
[[ "${USE_SYSLOG}" == "true" ]] && logger -t torrent-mover "[INFO] $*"
|
||||
}
|
||||
log_warn() {
|
||||
echo -e "[WARN] $(date '+%F %T') - $*" | tee -a "${LOG_FILE}" >&2
|
||||
[[ "${USE_SYSLOG}" == "true" ]] && logger -t torrent-mover "[WARN] $*"
|
||||
}
|
||||
log_error() {
|
||||
echo -e "[ERROR] $(date '+%F %T') - $*" | tee -a "${LOG_FILE}" >&2
|
||||
[[ "${USE_SYSLOG}" == "true" ]] && logger -t torrent-mover "[ERROR] $*"
|
||||
}
|
||||
|
||||
#################################
|
||||
# Error Handling & Notifications#
|
||||
#################################
|
||||
error_handler() {
|
||||
local lineno="$1"
|
||||
local msg="$2"
|
||||
log_error "Error on line ${lineno}: ${msg}"
|
||||
# Optionally send a notification (e.g., email)
|
||||
exit 1
|
||||
}
|
||||
trap 'error_handler ${LINENO} "$BASH_COMMAND"' ERR
|
||||
|
||||
##############################
|
||||
# Configuration & Validation #
|
||||
##############################
|
||||
@@ -79,6 +35,20 @@ if [[ -z "${TRANSMISSION_PATH_PREFIX:-}" || -z "${LOCAL_PATH_PREFIX:-}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Load modules
|
||||
for module in "${LIB_DIR}"/*.sh; do
|
||||
if [[ -f "$module" ]]; then
|
||||
source "$module"
|
||||
fi
|
||||
done
|
||||
|
||||
# Set defaults for new configuration options
|
||||
TORRENT_USER="${TORRENT_USER:-debian-transmission}"
|
||||
TORRENT_GROUP="${TORRENT_GROUP:-debian-transmission}"
|
||||
MAX_RETRY_ATTEMPTS="${MAX_RETRY_ATTEMPTS:-3}"
|
||||
RETRY_WAIT_TIME="${RETRY_WAIT_TIME:-15}"
|
||||
|
||||
# Enable DEBUG mode if set in config
|
||||
if [[ "${LOG_LEVEL}" == "DEBUG" ]]; then
|
||||
DEBUG=1
|
||||
fi
|
||||
@@ -89,313 +59,10 @@ if [[ -n "${STORAGE_DIRS}" ]]; then
|
||||
IFS=',' read -ra STORAGE_DIRS_ARRAY <<< "${STORAGE_DIRS}"
|
||||
fi
|
||||
|
||||
##############################
|
||||
# Helper & Utility Functions #
|
||||
##############################
|
||||
|
||||
# translate_source: Converts the Transmission‑reported path into the local path.
|
||||
translate_source() {
|
||||
local src="$1"
|
||||
echo "${src/#${TRANSMISSION_PATH_PREFIX}/${LOCAL_PATH_PREFIX}}"
|
||||
}
|
||||
|
||||
# parse_args: Processes command‑line options.
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
--interactive) INTERACTIVE=1; shift ;;
|
||||
--cache-warmup) CACHE_WARMUP=1; shift ;;
|
||||
--debug) DEBUG=1; shift ;;
|
||||
--help)
|
||||
echo "Usage: $0 [--dry-run] [--interactive] [--cache-warmup] [--debug]" >&2
|
||||
exit 0
|
||||
;;
|
||||
*) echo "Invalid option: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# check_dependencies: Ensures required commands are available.
|
||||
check_dependencies() {
|
||||
local deps=("transmission-remote" "unrar" "unzip" "7z" "parallel" "bc")
|
||||
for dep in "${deps[@]}"; do
|
||||
command -v "${dep}" >/dev/null 2>&1 || { log_error "Missing dependency: ${dep}"; exit 1; }
|
||||
done
|
||||
}
|
||||
|
||||
# check_disk_usage: Warn if disk usage is over 90%.
|
||||
declare -A CHECKED_MOUNTS=()
|
||||
check_disk_usage() {
|
||||
local dir="$1"
|
||||
[[ -z "${dir}" ]] && return
|
||||
if ! df -P "${dir}" &>/dev/null; then
|
||||
log_warn "Directory not found: ${dir}"
|
||||
return
|
||||
fi
|
||||
local mount_point
|
||||
mount_point=$(df -P "${dir}" | awk 'NR==2 {print $6}')
|
||||
[[ -z "${mount_point}" ]] && return
|
||||
if [[ -z "${CHECKED_MOUNTS["${mount_point}"]+x}" ]]; then
|
||||
local usage
|
||||
usage=$(df -P "${dir}" | awk 'NR==2 {sub(/%/, "", $5); print $5}')
|
||||
if (( usage >= 90 )); then
|
||||
log_warn "Storage warning: ${mount_point} at ${usage}% capacity"
|
||||
fi
|
||||
CHECKED_MOUNTS["${mount_point}"]=1
|
||||
fi
|
||||
}
|
||||
|
||||
# init_checksum_db: Initializes the checksum database.
|
||||
init_checksum_db() {
|
||||
mkdir -p "$(dirname "${CHECKSUM_DB}")"
|
||||
touch "${CHECKSUM_DB}" || { log_error "Could not create ${CHECKSUM_DB}"; exit 1; }
|
||||
chmod 600 "${CHECKSUM_DB}"
|
||||
}
|
||||
|
||||
# record_checksums: Generates checksums for files in given directories.
|
||||
record_checksums() {
|
||||
log_info "Generating checksums with ${PARALLEL_THREADS:-$(nproc)} threads"
|
||||
find "$@" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -print0 | \
|
||||
parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | sort > "${CHECKSUM_DB}.tmp"
|
||||
mv "${CHECKSUM_DB}.tmp" "${CHECKSUM_DB}"
|
||||
}
|
||||
|
||||
# file_metadata: Returns an md5 hash for file metadata.
|
||||
file_metadata() {
|
||||
find "$1" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort | awk '{print $1}'
|
||||
}
|
||||
|
||||
# files_need_processing: Checks if the source files need processing.
|
||||
files_need_processing() {
|
||||
local src="$1"
|
||||
shift
|
||||
local targets=("$@")
|
||||
|
||||
if [[ ! -d "${src}" ]]; then
|
||||
log_warn "Source directory missing: ${src}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "=== FILE VERIFICATION DEBUG START ==="
|
||||
log_info "Source directory: ${src}"
|
||||
log_info "Verification targets: ${targets[*]}"
|
||||
|
||||
local empty_target_found=0
|
||||
for target in "${targets[@]}"; do
|
||||
if [[ ! -d "${target}" ]]; then
|
||||
log_info "Target missing: ${target}"
|
||||
empty_target_found=1
|
||||
continue
|
||||
fi
|
||||
|
||||
local file_count
|
||||
file_count=$(find "${target}" -mindepth 1 -maxdepth 1 -print | wc -l)
|
||||
log_debug "File count for target ${target}: ${file_count}"
|
||||
if [[ "${file_count}" -eq 0 ]]; then
|
||||
log_info "Empty target directory: ${target}"
|
||||
empty_target_found=1
|
||||
else
|
||||
log_info "Target contains ${file_count} items: ${target}"
|
||||
log_info "First 5 items:"
|
||||
find "${target}" -mindepth 1 -maxdepth 1 | head -n 5 | while read -r item; do
|
||||
log_info " - ${item##*/}"
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${empty_target_found}" -eq 1 ]]; then
|
||||
log_info "Empty target detected - processing needed"
|
||||
log_info "=== FILE VERIFICATION DEBUG END ==="
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "Generating source checksums..."
|
||||
local src_checksums
|
||||
src_checksums=$(find "${src}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
|
||||
log_info "First 5 source checksums:"
|
||||
echo "${src_checksums}" | head -n 5 | while read -r line; do
|
||||
log_info " ${line}"
|
||||
done
|
||||
|
||||
local match_found=0
|
||||
for target in "${targets[@]}"; do
|
||||
log_info "Checking against target: ${target}"
|
||||
log_info "Generating target checksums..."
|
||||
local target_checksums
|
||||
target_checksums=$(find "${target}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
|
||||
log_info "First 5 target checksums:"
|
||||
echo "${target_checksums}" | head -n 5 | while read -r line; do
|
||||
log_info " ${line}"
|
||||
done
|
||||
|
||||
if diff <(echo "${src_checksums}") <(echo "${target_checksums}") >/dev/null; then
|
||||
log_info "Exact checksum match found in: ${target}"
|
||||
match_found=1
|
||||
break
|
||||
else
|
||||
log_info "No match in: ${target}"
|
||||
fi
|
||||
done
|
||||
|
||||
log_info "=== FILE VERIFICATION DEBUG END ==="
|
||||
[[ "${match_found}" -eq 1 ]] && return 1 || return 0
|
||||
}
|
||||
|
||||
# warm_cache: Pre-calculates checksums for storage directories.
|
||||
warm_cache() {
|
||||
log_info "Starting cache warmup for Movies..."
|
||||
local targets=("${DIR_MOVIES_DST}" "${STORAGE_DIRS_ARRAY[@]}")
|
||||
record_checksums "${targets[@]}"
|
||||
log_info "Cache warmup completed. Checksums stored in ${CHECKSUM_DB}"
|
||||
}
|
||||
|
||||
# is_processed: Checks if the torrent (by hash) has already been processed.
|
||||
is_processed() {
|
||||
grep -q "^${1}$" "${PROCESSED_LOG}" 2>/dev/null
|
||||
}
|
||||
|
||||
# mark_processed: Records a processed torrent.
|
||||
mark_processed() {
|
||||
echo "${1}" >> "${PROCESSED_LOG}"
|
||||
}
|
||||
|
||||
# get_destination: Maps a source directory to a destination directory based on keywords.
|
||||
declare -A PATH_CACHE
|
||||
get_destination() {
|
||||
local source_path="$1"
|
||||
if [[ -n "${PATH_CACHE["${source_path}"]+x}" ]]; then
|
||||
echo "${PATH_CACHE["${source_path}"]}"
|
||||
return
|
||||
fi
|
||||
log_info "Analyzing path: ${source_path}"
|
||||
local destination
|
||||
case "${source_path,,}" in
|
||||
*games*) destination="${DIR_GAMES_DST}";;
|
||||
*apps*) destination="${DIR_APPS_DST}";;
|
||||
*movies*) destination="${DIR_MOVIES_DST}";;
|
||||
*books*) destination="${DIR_BOOKS_DST}";;
|
||||
*) destination="${DEFAULT_DST}";;
|
||||
esac
|
||||
log_info "Mapped to: ${destination}"
|
||||
PATH_CACHE["${source_path}"]="${destination}"
|
||||
echo "${destination}"
|
||||
}
|
||||
|
||||
######################################
|
||||
# Improved Archive Extraction Handler #
|
||||
######################################
|
||||
# For each archive found in the source directory, create a subdirectory in the destination
|
||||
# named after the archive (without its extension) and extract into that subdirectory.
|
||||
# IMPORTANT: The archive is now retained in the source, so it will remain until the ratio
|
||||
# limits are reached and Transmission removes the torrent data.
|
||||
handle_archives() {
|
||||
local src="$1" dst="$2"
|
||||
find "${src}" -type f \( -iname "*.rar" -o -iname "*.zip" -o -iname "*.7z" \) | while read -r arch; do
|
||||
log_info "Extracting archive: ${arch}"
|
||||
local base
|
||||
base=$(basename "${arch}")
|
||||
local subdir="${dst}/${base%.*}"
|
||||
mkdir -p "${subdir}" || { log_error "Failed to create subdirectory ${subdir}"; continue; }
|
||||
case "${arch##*.}" in
|
||||
rar)
|
||||
unrar x -o- "${arch}" "${subdir}" || { log_error "unrar failed for ${arch}"; continue; }
|
||||
;;
|
||||
zip)
|
||||
unzip -o "${arch}" -d "${subdir}" || { log_error "unzip failed for ${arch}"; continue; }
|
||||
;;
|
||||
7z)
|
||||
7z x "${arch}" -o"${subdir}" || { log_error "7z extraction failed for ${arch}"; continue; }
|
||||
;;
|
||||
esac
|
||||
log_info "Archive ${arch} retained in source until ratio limits are reached."
|
||||
done
|
||||
}
|
||||
|
||||
# move_files: Moves files using parallel processing if enabled.
|
||||
move_files() {
|
||||
if (( PARALLEL_PROCESSING )); then
|
||||
parallel -j ${PARALLEL_THREADS:-$(nproc)} mv {} "${1}" ::: "${2}"/*
|
||||
else
|
||||
mv "${2}"/* "${1}"
|
||||
fi
|
||||
}
|
||||
|
||||
# copy_files: Copies files using parallel processing if enabled.
|
||||
copy_files() {
|
||||
if (( PARALLEL_PROCESSING )); then
|
||||
parallel -j ${PARALLEL_THREADS:-$(nproc)} cp -r {} "${1}" ::: "${2}"/*
|
||||
else
|
||||
cp -r "${2}"/* "${1}"
|
||||
fi
|
||||
}
|
||||
|
||||
# process_copy: Validates directories, then copies/moves files from source to destination.
|
||||
# Optionally verifies integrity after transfer if CHECK_TRANSFER_INTEGRITY is "true".
|
||||
process_copy() {
|
||||
local id="$1" hash="$2" src="$3" dst="$4"
|
||||
if [[ ! -d "${src}" ]]; then
|
||||
log_error "Source directory missing: ${src}"
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -d "${dst}" ]]; then
|
||||
log_info "Creating destination directory: ${dst}"
|
||||
mkdir -p "${dst}" || { log_error "Failed to create directory: ${dst}"; return 1; }
|
||||
chmod 775 "${dst}"
|
||||
chown debian-transmission:debian-transmission "${dst}"
|
||||
fi
|
||||
if [[ ! -w "${dst}" ]]; then
|
||||
log_error "No write permissions for: ${dst}"
|
||||
return 1
|
||||
fi
|
||||
if (( DRY_RUN )); then
|
||||
log_info "[DRY RUN] Would process torrent ${id}:"
|
||||
log_info " - Copy files from ${src} to ${dst}"
|
||||
log_info " - File count: $(find "${src}" -maxdepth 1 -type f | wc -l)"
|
||||
return
|
||||
fi
|
||||
handle_archives "${src}" "${dst}"
|
||||
case "${COPY_MODE}" in
|
||||
move)
|
||||
log_info "Moving files from ${src} to ${dst}"
|
||||
move_files "${dst}" "${src}"
|
||||
;;
|
||||
copy)
|
||||
log_info "Copying files from ${src} to ${dst}"
|
||||
copy_files "${dst}" "${src}"
|
||||
;;
|
||||
esac
|
||||
if [ $? -eq 0 ]; then
|
||||
if [[ "${CHECK_TRANSFER_INTEGRITY}" == "true" ]]; then
|
||||
log_info "Verifying integrity of transferred files..."
|
||||
local src_checksum target_checksum
|
||||
src_checksum=$(find "${src}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
|
||||
target_checksum=$(find "${dst}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
|
||||
if diff <(echo "${src_checksum}") <(echo "${target_checksum}") >/dev/null; then
|
||||
log_info "Integrity check passed."
|
||||
else
|
||||
log_error "Integrity check FAILED for ${src}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
log_info "Transfer completed successfully"
|
||||
mark_processed "${hash}"
|
||||
else
|
||||
log_error "Transfer failed for ${src}"
|
||||
fi
|
||||
}
|
||||
|
||||
# process_removal: Removes a torrent via Transmission.
|
||||
process_removal() {
|
||||
local id="$1"
|
||||
if (( DRY_RUN )); then
|
||||
log_info "[DRY RUN] Would remove torrent ${id}"
|
||||
return
|
||||
fi
|
||||
transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
|
||||
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" \
|
||||
-t "${id}" --remove-and-delete
|
||||
}
|
||||
#################################
|
||||
# Error Handling & Notifications#
|
||||
#################################
|
||||
trap 'error_handler ${LINENO} "$BASH_COMMAND"' ERR
|
||||
|
||||
#################
|
||||
# Main Function #
|
||||
@@ -411,16 +78,12 @@ main() {
|
||||
"${DIR_BOOKS_DST}"
|
||||
"${DEFAULT_DST}"
|
||||
)
|
||||
for dir in "${REQUIRED_DIRS[@]}"; do
|
||||
if [[ ! -d "${dir}" ]]; then
|
||||
log_error "Directory missing: ${dir}"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -w "${dir}" ]]; then
|
||||
log_error "Write permission denied: ${dir}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Add optional directories if defined
|
||||
[[ -n "${DIR_TV_DST}" ]] && REQUIRED_DIRS+=("${DIR_TV_DST}")
|
||||
[[ -n "${DIR_MUSIC_DST}" ]] && REQUIRED_DIRS+=("${DIR_MUSIC_DST}")
|
||||
|
||||
validate_directories "${REQUIRED_DIRS[@]}" || exit 1
|
||||
|
||||
init_checksum_db
|
||||
|
||||
@@ -429,14 +92,14 @@ main() {
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log_info "Starting processing"
|
||||
log_info "Starting processing with user: ${TORRENT_USER}"
|
||||
declare -A warned_dirs=()
|
||||
transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
|
||||
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -l | awk 'NR>1 && $1 ~ /^[0-9]+$/ {print $1}' | while read -r id; do
|
||||
|
||||
|
||||
# Get list of torrents from Transmission
|
||||
get_torrents | while read -r id; do
|
||||
local info
|
||||
info=$(transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
|
||||
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -t "${id}" -i)
|
||||
info=$(get_torrent_info "${id}")
|
||||
|
||||
local hash
|
||||
hash=$(grep "Hash:" <<< "${info}" | awk '{print $2}')
|
||||
local ratio
|
||||
@@ -470,8 +133,15 @@ main() {
|
||||
fi
|
||||
local targets=("${dst}")
|
||||
case "${dst}" in
|
||||
"${DIR_MOVIES_DST}") targets+=("${STORAGE_DIRS_ARRAY[@]}");;
|
||||
"${DIR_MOVIES_DST}")
|
||||
targets+=("${STORAGE_DIRS_ARRAY[@]}")
|
||||
;;
|
||||
"${DIR_TV_DST}")
|
||||
# If there are TV storage dirs, include them
|
||||
[[ -n "${STORAGE_TV_DIRS}" ]] && IFS=',' read -ra TV_DIRS <<< "${STORAGE_TV_DIRS}" && targets+=("${TV_DIRS[@]}")
|
||||
;;
|
||||
esac
|
||||
|
||||
if ! files_need_processing "${dir}" "${targets[@]}"; then
|
||||
log_info "Skipping copy - files already exist in:"
|
||||
for target in "${targets[@]}"; do
|
||||
@@ -489,11 +159,13 @@ main() {
|
||||
fi
|
||||
done
|
||||
|
||||
check_disk_usage "${DIR_GAMES_DST}"
|
||||
check_disk_usage "${DIR_APPS_DST}"
|
||||
check_disk_usage "${DIR_MOVIES_DST}"
|
||||
check_disk_usage "${DIR_BOOKS_DST}"
|
||||
check_disk_usage "${DEFAULT_DST}"
|
||||
# Check disk usage for all directories
|
||||
for dir in "${REQUIRED_DIRS[@]}"; do
|
||||
check_disk_usage "${dir}"
|
||||
done
|
||||
for dir in "${STORAGE_DIRS_ARRAY[@]}"; do
|
||||
check_disk_usage "${dir}"
|
||||
done
|
||||
}
|
||||
|
||||
######################
|
||||
@@ -506,4 +178,4 @@ if (( INTERACTIVE )); then
|
||||
[[ "${choice}" =~ ^[Yy]$ ]] || exit 0
|
||||
fi
|
||||
|
||||
main
|
||||
main
|
||||
Reference in New Issue
Block a user