From 7c65712b16a8bc8663632a603956e62078e041da Mon Sep 17 00:00:00 2001 From: masterdraco Date: Tue, 25 Feb 2025 00:38:20 +0100 Subject: [PATCH] a lot of debugging later --- etc/torrent/mover.conf | 11 +- usr/local/bin/torrent-mover | 276 ++++++++++++++++++++++++------------ 2 files changed, 192 insertions(+), 95 deletions(-) diff --git a/etc/torrent/mover.conf b/etc/torrent/mover.conf index 2d3d225..b57e7b3 100644 --- a/etc/torrent/mover.conf +++ b/etc/torrent/mover.conf @@ -18,8 +18,12 @@ DEFAULT_DST="/mnt/dsnas1/Other" # Storage directories (comma-separated) STORAGE_DIRS="/mnt/dsnas/Movies" +# Path mapping +TRANSMISSION_PATH_PREFIX="/downloads" +LOCAL_PATH_PREFIX="/mnt/dsnas2" + # Performance settings -PARALLEL_THREADS="16" # Match CPU core count +PARALLEL_THREADS="32" # Match CPU core count PARALLEL_PROCESSING=1 # Operation mode @@ -29,3 +33,8 @@ CHECKSUM_DB="/var/lib/torrent/checksums.db" # System settings LOG_FILE="/var/log/torrent_mover.log" + +# Auto-create directories +mkdir -p "${DIR_GAMES_DST}" "${DIR_APPS_DST}" \ + "${DIR_MOVIES_DST}" "${DIR_BOOKS_DST}" \ + "${DEFAULT_DST}" 2>/dev/null || true diff --git a/usr/local/bin/torrent-mover b/usr/local/bin/torrent-mover index 503f426..e75b10f 100755 --- a/usr/local/bin/torrent-mover +++ b/usr/local/bin/torrent-mover @@ -1,21 +1,19 @@ #!/bin/bash -# Torrent Mover v5.3 - Singleton Implementation +# Torrent Mover v7.2 - Final Debugged Version -# Singleton pattern +# Singleton pattern implementation LOCK_FILE="/var/lock/torrent-mover.lock" MAX_AGE=300 # 5 minutes in seconds # Check for existing lock if [ -f "${LOCK_FILE}" ]; then PID=$(cat "${LOCK_FILE}") - - # Check if process exists - if ps -p "${PID}" > /dev/null 2>&1; then - echo "Already running (PID: ${PID}), exiting." + if ps -p "${PID}" >/dev/null 2>&1; then + echo "torrent-mover already running (PID: ${PID}), exiting." exit 1 else - # Check lock file age - if [ $(($(date +%s) - $(date -r "${LOCK_FILE}" +%s))) -lt ${MAX_AGE} ]; then + LOCK_AGE=$(($(date +%s) - $(stat -c %Y "${LOCK_FILE}"))) + if [ "${LOCK_AGE}" -lt "${MAX_AGE}" ]; then echo "Recent crash detected, waiting..." exit 1 fi @@ -24,7 +22,6 @@ if [ -f "${LOCK_FILE}" ]; then fi fi -# Create new lock echo $$ > "${LOCK_FILE}" trap 'rm -f "${LOCK_FILE}"' EXIT TERM INT @@ -34,8 +31,20 @@ set -o pipefail # Load configuration CONFIG_FILE="/etc/torrent/mover.conf" +if [[ ! -f "${CONFIG_FILE}" ]]; then + echo "FATAL: Configuration file missing: ${CONFIG_FILE}" >&2 + exit 1 +fi source "${CONFIG_FILE}" +# --- New: Source Path Translation Function --- +translate_source() { + local src="$1" + # Replace the Transmission reported prefix with the local prefix. + # Example: /downloads/Books becomes /mnt/dsnas2/Books. + echo "${src/#${TRANSMISSION_PATH_PREFIX}/${LOCAL_PATH_PREFIX}}" +} + # Runtime flags DRY_RUN=0 INTERACTIVE=0 @@ -71,21 +80,19 @@ check_dependencies() { # Disk usage monitoring declare -A CHECKED_MOUNTS=() - check_disk_usage() { local dir="$1" [[ -z "${dir}" ]] && return - if ! df -P "${dir}" &>/dev/null; then log_warn "Directory not found: ${dir}" return fi - - local mount_point=$(df -P "${dir}" | awk 'NR==2 {print $6}') + local mount_point + mount_point=$(df -P "${dir}" | awk 'NR==2 {print $6}') [[ -z "${mount_point}" ]] && return - if [[ -z "${CHECKED_MOUNTS["${mount_point}"]+x}" ]]; then - local usage=$(df -P "${dir}" | awk 'NR==2 {sub(/%/, "", $5); print $5}') + local usage + usage=$(df -P "${dir}" | awk 'NR==2 {sub(/%/, "", $5); print $5}') if (( usage >= 90 )); then log_warn "Storage warning: ${mount_point} at ${usage}% capacity" fi @@ -102,42 +109,88 @@ init_checksum_db() { record_checksums() { log_info "Generating checksums with ${PARALLEL_THREADS:-$(nproc)} threads" - find "$@" -type f \( -iname "*.nfo" -o -iname "*.sfv" \) -prune -o -type f -print0 | \ - parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | \ - sort > "${CHECKSUM_DB}.tmp" + find "$@" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -print0 | \ + parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | sort > "${CHECKSUM_DB}.tmp" mv "${CHECKSUM_DB}.tmp" "${CHECKSUM_DB}" } file_metadata() { - find "$1" -type f \( -iname "*.nfo" -o -iname "*.sfv" \) -prune -o -type f -printf "%s %T@ %p\n" | \ - sort | \ - md5sum | \ - awk '{print $1}' + find "$1" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort | awk '{print $1}' } files_need_processing() { - local src="$1" shift + local src="$1" + shift local targets=("$@") - [[ ! -d "${src}" ]] && return 1 + if [[ ! -d "${src}" ]]; then + log_warn "Source directory missing: ${src}" + return 1 + fi - local src_meta=$(file_metadata "${src}") - + log_info "=== FILE VERIFICATION DEBUG START ===" + log_info "Source directory: ${src}" + log_info "Verification targets: ${targets[*]}" + + local empty_target_found=0 for target in "${targets[@]}"; do - [[ ! -d "${target}" ]] && continue - local target_meta=$(file_metadata "${target}") - [[ "${src_meta}" == "${target_meta}" ]] && return 1 + if [[ ! -d "${target}" ]]; then + log_info "Target missing: ${target}" + empty_target_found=1 + continue + fi + + local file_count + file_count=$(find "${target}" -mindepth 1 -maxdepth 1 -print | wc -l) + log_info "File count for target ${target}: ${file_count}" + if [[ "${file_count}" -eq 0 ]]; then + log_info "Empty target directory: ${target}" + empty_target_found=1 + else + log_info "Target contains ${file_count} items: ${target}" + log_info "First 5 items:" + find "${target}" -mindepth 1 -maxdepth 1 | head -n 5 | while read -r item; do + log_info " - ${item##*/}" + done + fi done - local src_checksums=$(find "${src}" -type f \( -iname "*.nfo" -o -iname "*.sfv" \) -prune -o -type f -exec md5sum {} \; | sort) - - for target in "${targets[@]}"; do - [[ ! -d "${target}" ]] && continue - local target_checksums=$(find "${target}" -type f \( -iname "*.nfo" -o -iname "*.sfv" \) -prune -o -type f -exec md5sum {} \; | sort) - diff <(echo "${src_checksums}") <(echo "${target_checksums}") >/dev/null && return 1 + if [[ "${empty_target_found}" -eq 1 ]]; then + log_info "Empty target detected - processing needed" + log_info "=== FILE VERIFICATION DEBUG END ===" + return 0 + fi + + log_info "Generating source checksums..." + local src_checksums + src_checksums=$(find "${src}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort) + log_info "First 5 source checksums:" + echo "${src_checksums}" | head -n 5 | while read -r line; do + log_info " ${line}" done - - return 0 + + local match_found=0 + for target in "${targets[@]}"; do + log_info "Checking against target: ${target}" + log_info "Generating target checksums..." + local target_checksums + target_checksums=$(find "${target}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort) + log_info "First 5 target checksums:" + echo "${target_checksums}" | head -n 5 | while read -r line; do + log_info " ${line}" + done + + if diff <(echo "${src_checksums}") <(echo "${target_checksums}") >/dev/null; then + log_info "Exact checksum match found in: ${target}" + match_found=1 + break + else + log_info "No match in: ${target}" + fi + done + + log_info "=== FILE VERIFICATION DEBUG END ===" + [[ "${match_found}" -eq 1 ]] && return 1 || return 0 } warm_cache() { @@ -155,20 +208,30 @@ mark_processed() { echo "${1}" >> "${PROCESSED_LOG}" } +declare -A PATH_CACHE get_destination() { - case "${1}" in - *Games*) echo "${DIR_GAMES_DST}";; - *Apps*) echo "${DIR_APPS_DST}";; - *Movies*) echo "${DIR_MOVIES_DST}";; - *Books*) echo "${DIR_BOOKS_DST}";; - *) echo "${DEFAULT_DST}";; + local source_path="$1" + if [[ -n "${PATH_CACHE["${source_path}"]+x}" ]]; then + echo "${PATH_CACHE["${source_path}"]}" + return + fi + log_info "Analyzing path: ${source_path}" + local destination + case "${source_path,,}" in + *games*) destination="${DIR_GAMES_DST}";; + *apps*) destination="${DIR_APPS_DST}";; + *movies*) destination="${DIR_MOVIES_DST}";; + *books*) destination="${DIR_BOOKS_DST}";; + *) destination="${DEFAULT_DST}";; esac + log_info "Mapped to: ${destination}" + PATH_CACHE["${source_path}"]="${destination}" + echo "${destination}" } handle_archives() { local src="$1" dst="$2" - find "${src}" -type f \( -iname "*.rar" -o -iname "*.zip" -o -iname "*.7z" \) | \ - while read -r arch; do + find "${src}" -type f \( -iname "*.rar" -o -iname "*.zip" -o -iname "*.7z" \) | while read -r arch; do log_info "Extracting ${arch##*/}" case "${arch##*.}" in rar) unrar x -o- "${arch}" "${dst}";; @@ -196,30 +259,54 @@ copy_files() { process_copy() { local id="$1" hash="$2" src="$3" dst="$4" - + if [[ ! -d "${src}" ]]; then + log_error "Source directory missing: ${src}" + return 1 + fi + if [[ ! -d "${dst}" ]]; then + log_info "Creating destination directory: ${dst}" + mkdir -p "${dst}" || { + log_error "Failed to create directory: ${dst}" + return 1 + } + chmod 775 "${dst}" + chown debian-transmission:debian-transmission "${dst}" + fi + if [[ ! -w "${dst}" ]]; then + log_error "No write permissions for: ${dst}" + return 1 + fi if (( DRY_RUN )); then log_info "[DRY RUN] Would process torrent ${id}:" log_info " - Copy files from ${src} to ${dst}" + log_info " - File count: $(find "${src}" -maxdepth 1 -type f | wc -l)" return fi - - mkdir -p "${dst}" handle_archives "${src}" "${dst}" - case "${COPY_MODE}" in - move) move_files "${dst}" "${src}";; - copy) copy_files "${dst}" "${src}";; + move) + log_info "Moving files from ${src} to ${dst}" + move_files "${dst}" "${src}" + ;; + copy) + log_info "Copying files from ${src} to ${dst}" + copy_files "${dst}" "${src}" + ;; esac + if [ $? -eq 0 ]; then + log_info "Transfer completed successfully" + mark_processed "${hash}" + else + log_error "Transfer failed for ${src}" + fi } process_removal() { local id="$1" - if (( DRY_RUN )); then log_info "[DRY RUN] Would remove torrent ${id}" return fi - transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \ -n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" \ -t "${id}" --remove-and-delete @@ -227,56 +314,65 @@ process_removal() { main() { check_dependencies + declare -a REQUIRED_DIRS=( + "${DIR_GAMES_DST}" + "${DIR_APPS_DST}" + "${DIR_MOVIES_DST}" + "${DIR_BOOKS_DST}" + "${DEFAULT_DST}" + ) + for dir in "${REQUIRED_DIRS[@]}"; do + if [[ ! -d "${dir}" ]]; then + log_error "Directory missing: ${dir}" + exit 1 + fi + if [[ ! -w "${dir}" ]]; then + log_error "Write permission denied: ${dir}" + exit 1 + fi + done init_checksum_db - if (( CACHE_WARMUP )); then warm_cache exit 0 fi - log_info "Starting processing" declare -A warned_dirs=() - transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \ - -n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -l | \ - awk 'NR>1 && $1 ~ /^[0-9]+$/ {print $1}' | \ - while read -r id; do - - local info=$(transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \ + -n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -l | awk 'NR>1 && $1 ~ /^[0-9]+$/ {print $1}' | while read -r id; do + local info + info=$(transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \ -n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -t "${id}" -i) - local hash=$(grep "Hash:" <<< "${info}" | awk '{print $2}') - - # Sanitize numeric values with fallbacks - local ratio=$(grep "Ratio:" <<< "${info}" | awk '{print $2 == "None" ? 0 : $2}' | tr -cd '0-9.') - ratio=${ratio:-0} # Handle empty values - - local time=$(grep "Seeding Time:" <<< "${info}" | awk '{print $3 == "None" ? 0 : $3}' | tr -cd '0-9.') - time=${time:-0} # Handle empty values - - local percent_done=$(grep "Percent Done:" <<< "${info}" | awk '{gsub(/%/, ""); print $3 == "None" ? 0 : $3}') - percent_done=${percent_done:-0} # Handle empty values - - local dir=$(grep "Location:" <<< "${info}" | cut -d' ' -f4-) - local dst=$(get_destination "${dir}") - - # Initialize warning tracking + local hash + hash=$(grep "Hash:" <<< "${info}" | awk '{print $2}') + local ratio + ratio=$(grep "Ratio:" <<< "${info}" | awk '{print $2 == "None" ? 0 : $2}' | tr -cd '0-9.') + ratio=${ratio:-0} + local time + time=$(grep "Seeding Time:" <<< "${info}" | awk '{print $3 == "None" ? 0 : $3}' | tr -cd '0-9.') + time=${time:-0} + local percent_done + percent_done=$(grep "Percent Done:" <<< "${info}" | awk '{gsub(/%/, ""); print $3 == "None" ? 0 : $3}') + percent_done=${percent_done:-0} + # Extract and translate download location. + local reported_dir + reported_dir=$(grep -i "Location:" <<< "${info}" | awk -F": " '{print $2}' | xargs) + local dir + dir=$(translate_source "${reported_dir}") + log_info "Torrent source directory reported: '${reported_dir}' translated to '${dir}'" + local dst + dst=$(get_destination "${dir}") [[ -z "${warned_dirs["${dir}"]+x}" ]] && warned_dirs["${dir}"]=0 - - # 1. Handle completed downloads if (( $(bc <<< "${percent_done} >= 100") )) && ! is_processed "${hash}"; then log_info "Processing completed torrent ${id} (${percent_done}% done)" - if [[ "${dst}" == "${DEFAULT_DST}" ]] && (( warned_dirs["${dir}"] == 0 )); then log_warn "Using default destination for: ${dir}" warned_dirs["${dir}"]=1 fi - - # Determine check targets local targets=("${dst}") - if [[ "${dst}" == "${DIR_MOVIES_DST}" ]]; then - targets+=("${STORAGE_DIRS_ARRAY[@]}") - fi - + case "${dst}" in + "${DIR_MOVIES_DST}") targets+=("${STORAGE_DIRS_ARRAY[@]}");; + esac if ! files_need_processing "${dir}" "${targets[@]}"; then log_info "Skipping copy - files already exist in:" for target in "${targets[@]}"; do @@ -285,18 +381,12 @@ main() { else process_copy "${id}" "${hash}" "${dir}" "${dst}" fi - - mark_processed "${hash}" fi - - # 2. Handle seeding criteria if (( $(bc <<< "${ratio} >= ${SEED_RATIO}") )) || (( $(bc <<< "${time} >= ${SEED_TIME}") )); then log_info "Removing torrent ${id} (Ratio: ${ratio}, Time: ${time})" process_removal "${id}" fi done - - # Final disk checks check_disk_usage "${DIR_GAMES_DST}" check_disk_usage "${DIR_APPS_DST}" check_disk_usage "${DIR_MOVIES_DST}" @@ -304,7 +394,6 @@ main() { check_disk_usage "${DEFAULT_DST}" } -# Argument handling while [[ $# -gt 0 ]]; do case "$1" in --dry-run) DRY_RUN=1; shift ;; @@ -318,7 +407,6 @@ while [[ $# -gt 0 ]]; do esac done -# Execution if (( INTERACTIVE )); then read -rp "Confirm processing? (y/n) " choice [[ "${choice}" =~ ^[Yy]$ ]] || exit 0