masterdraco/torrent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

a lot of debugging later

Browse Source
This commit is contained in:
masterdraco 2025-02-25 00:38:20 +01:00
parent fce11134d8
commit 7c65712b16
2 changed files with 192 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
etc/torrent/mover.conf
View File

@ -18,8 +18,12 @@ DEFAULT_DST="/mnt/dsnas1/Other"
# Storage directories (comma-separated) # Storage directories (comma-separated)
STORAGE_DIRS="/mnt/dsnas/Movies" STORAGE_DIRS="/mnt/dsnas/Movies"
# Path mapping
TRANSMISSION_PATH_PREFIX="/downloads"
LOCAL_PATH_PREFIX="/mnt/dsnas2"
# Performance settings # Performance settings
PARALLEL_THREADS="16" # Match CPU core count PARALLEL_THREADS="32" # Match CPU core count
PARALLEL_PROCESSING=1 PARALLEL_PROCESSING=1
# Operation mode # Operation mode
@ -29,3 +33,8 @@ CHECKSUM_DB="/var/lib/torrent/checksums.db"
# System settings # System settings
LOG_FILE="/var/log/torrent_mover.log" LOG_FILE="/var/log/torrent_mover.log"
# Auto-create directories
mkdir -p "${DIR_GAMES_DST}" "${DIR_APPS_DST}" \
"${DIR_MOVIES_DST}" "${DIR_BOOKS_DST}" \
"${DEFAULT_DST}" 2>/dev/null || true

274
usr/local/bin/torrent-mover
View File

@ -1,21 +1,19 @@
#!/bin/bash #!/bin/bash
# Torrent Mover v5.3 - Singleton Implementation # Torrent Mover v7.2 - Final Debugged Version
# Singleton pattern # Singleton pattern implementation
LOCK_FILE="/var/lock/torrent-mover.lock" LOCK_FILE="/var/lock/torrent-mover.lock"
MAX_AGE=300 # 5 minutes in seconds MAX_AGE=300 # 5 minutes in seconds
# Check for existing lock # Check for existing lock
if [ -f "${LOCK_FILE}" ]; then if [ -f "${LOCK_FILE}" ]; then
PID=$(cat "${LOCK_FILE}") PID=$(cat "${LOCK_FILE}")
if ps -p "${PID}" >/dev/null 2>&1; then
# Check if process exists echo "torrent-mover already running (PID: ${PID}), exiting."
if ps -p "${PID}" > /dev/null 2>&1; then
echo "Already running (PID: ${PID}), exiting."
exit 1 exit 1
else else
# Check lock file age LOCK_AGE=$(($(date +%s) - $(stat -c %Y "${LOCK_FILE}")))
if [ $(($(date +%s) - $(date -r "${LOCK_FILE}" +%s))) -lt ${MAX_AGE} ]; then if [ "${LOCK_AGE}" -lt "${MAX_AGE}" ]; then
echo "Recent crash detected, waiting..." echo "Recent crash detected, waiting..."
exit 1 exit 1
fi fi
@ -24,7 +22,6 @@ if [ -f "${LOCK_FILE}" ]; then
fi fi
fi fi
# Create new lock
echo $$ > "${LOCK_FILE}" echo $$ > "${LOCK_FILE}"
trap 'rm -f "${LOCK_FILE}"' EXIT TERM INT trap 'rm -f "${LOCK_FILE}"' EXIT TERM INT
@ -34,8 +31,20 @@ set -o pipefail
# Load configuration # Load configuration
CONFIG_FILE="/etc/torrent/mover.conf" CONFIG_FILE="/etc/torrent/mover.conf"
if [[ ! -f "${CONFIG_FILE}" ]]; then
echo "FATAL: Configuration file missing: ${CONFIG_FILE}" >&2
exit 1
fi
source "${CONFIG_FILE}" source "${CONFIG_FILE}"
# --- New: Source Path Translation Function ---
translate_source() {
local src="$1"
# Replace the Transmission reported prefix with the local prefix.
# Example: /downloads/Books becomes /mnt/dsnas2/Books.
echo "${src/#${TRANSMISSION_PATH_PREFIX}/${LOCAL_PATH_PREFIX}}"
}
# Runtime flags # Runtime flags
DRY_RUN=0 DRY_RUN=0
INTERACTIVE=0 INTERACTIVE=0
@ -71,21 +80,19 @@ check_dependencies() {
# Disk usage monitoring # Disk usage monitoring
declare -A CHECKED_MOUNTS=() declare -A CHECKED_MOUNTS=()
check_disk_usage() { check_disk_usage() {
local dir="$1" local dir="$1"
[[ -z "${dir}" ]] && return [[ -z "${dir}" ]] && return
if ! df -P "${dir}" &>/dev/null; then if ! df -P "${dir}" &>/dev/null; then
log_warn "Directory not found: ${dir}" log_warn "Directory not found: ${dir}"
return return
fi fi
local mount_point
local mount_point=$(df -P "${dir}" | awk 'NR==2 {print $6}') mount_point=$(df -P "${dir}" | awk 'NR==2 {print $6}')
[[ -z "${mount_point}" ]] && return [[ -z "${mount_point}" ]] && return
if [[ -z "${CHECKED_MOUNTS["${mount_point}"]+x}" ]]; then if [[ -z "${CHECKED_MOUNTS["${mount_point}"]+x}" ]]; then
local usage=$(df -P "${dir}" | awk 'NR==2 {sub(/%/, "", $5); print $5}') local usage
usage=$(df -P "${dir}" | awk 'NR==2 {sub(/%/, "", $5); print $5}')
if (( usage >= 90 )); then if (( usage >= 90 )); then
log_warn "Storage warning: ${mount_point} at ${usage}% capacity" log_warn "Storage warning: ${mount_point} at ${usage}% capacity"
fi fi
@ -102,42 +109,88 @@ init_checksum_db() {
record_checksums() { record_checksums() {
log_info "Generating checksums with ${PARALLEL_THREADS:-$(nproc)} threads" log_info "Generating checksums with ${PARALLEL_THREADS:-$(nproc)} threads"
find "$@" -type f \( -iname "*.nfo" -o -iname "*.sfv" \) -prune -o -type f -print0 | \ find "$@" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -print0 | \
parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | \ parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | sort > "${CHECKSUM_DB}.tmp"
sort > "${CHECKSUM_DB}.tmp"
mv "${CHECKSUM_DB}.tmp" "${CHECKSUM_DB}" mv "${CHECKSUM_DB}.tmp" "${CHECKSUM_DB}"
} }
file_metadata() { file_metadata() {
find "$1" -type f \( -iname "*.nfo" -o -iname "*.sfv" \) -prune -o -type f -printf "%s %T@ %p\n" | \ find "$1" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort | awk '{print $1}'
sort | \
md5sum | \
awk '{print $1}'
} }
files_need_processing() { files_need_processing() {
local src="$1" shift local src="$1"
shift
local targets=("$@") local targets=("$@")
[[ ! -d "${src}" ]] && return 1 if [[ ! -d "${src}" ]]; then
log_warn "Source directory missing: ${src}"
return 1
fi
local src_meta=$(file_metadata "${src}") log_info "=== FILE VERIFICATION DEBUG START ==="
log_info "Source directory: ${src}"
log_info "Verification targets: ${targets[*]}"
local empty_target_found=0
for target in "${targets[@]}"; do for target in "${targets[@]}"; do
[[ ! -d "${target}" ]] && continue if [[ ! -d "${target}" ]]; then
local target_meta=$(file_metadata "${target}") log_info "Target missing: ${target}"
[[ "${src_meta}" == "${target_meta}" ]] && return 1 empty_target_found=1
done continue
fi
local src_checksums=$(find "${src}" -type f \( -iname "*.nfo" -o -iname "*.sfv" \) -prune -o -type f -exec md5sum {} \; | sort)
local file_count
for target in "${targets[@]}"; do file_count=$(find "${target}" -mindepth 1 -maxdepth 1 -print | wc -l)
[[ ! -d "${target}" ]] && continue log_info "File count for target ${target}: ${file_count}"
local target_checksums=$(find "${target}" -type f \( -iname "*.nfo" -o -iname "*.sfv" \) -prune -o -type f -exec md5sum {} \; | sort) if [[ "${file_count}" -eq 0 ]]; then
diff <(echo "${src_checksums}") <(echo "${target_checksums}") >/dev/null && return 1 log_info "Empty target directory: ${target}"
empty_target_found=1
else
log_info "Target contains ${file_count} items: ${target}"
log_info "First 5 items:"
find "${target}" -mindepth 1 -maxdepth 1 | head -n 5 | while read -r item; do
log_info " - ${item##*/}"
done
fi
done done
if [[ "${empty_target_found}" -eq 1 ]]; then
log_info "Empty target detected - processing needed"
log_info "=== FILE VERIFICATION DEBUG END ==="
return 0 return 0
fi
log_info "Generating source checksums..."
local src_checksums
src_checksums=$(find "${src}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
log_info "First 5 source checksums:"
echo "${src_checksums}" | head -n 5 | while read -r line; do
log_info " ${line}"
done
local match_found=0
for target in "${targets[@]}"; do
log_info "Checking against target: ${target}"
log_info "Generating target checksums..."
local target_checksums
target_checksums=$(find "${target}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec md5sum {} \; | sort)
log_info "First 5 target checksums:"
echo "${target_checksums}" | head -n 5 | while read -r line; do
log_info " ${line}"
done
if diff <(echo "${src_checksums}") <(echo "${target_checksums}") >/dev/null; then
log_info "Exact checksum match found in: ${target}"
match_found=1
break
else
log_info "No match in: ${target}"
fi
done
log_info "=== FILE VERIFICATION DEBUG END ==="
[[ "${match_found}" -eq 1 ]] && return 1 || return 0
} }
warm_cache() { warm_cache() {
@ -155,20 +208,30 @@ mark_processed() {
echo "${1}" >> "${PROCESSED_LOG}" echo "${1}" >> "${PROCESSED_LOG}"
} }
declare -A PATH_CACHE
get_destination() { get_destination() {
case "${1}" in local source_path="$1"
*Games*) echo "${DIR_GAMES_DST}";; if [[ -n "${PATH_CACHE["${source_path}"]+x}" ]]; then
*Apps*) echo "${DIR_APPS_DST}";; echo "${PATH_CACHE["${source_path}"]}"
*Movies*) echo "${DIR_MOVIES_DST}";; return
*Books*) echo "${DIR_BOOKS_DST}";; fi
*) echo "${DEFAULT_DST}";; log_info "Analyzing path: ${source_path}"
local destination
case "${source_path,,}" in
*games*) destination="${DIR_GAMES_DST}";;
*apps*) destination="${DIR_APPS_DST}";;
*movies*) destination="${DIR_MOVIES_DST}";;
*books*) destination="${DIR_BOOKS_DST}";;
*) destination="${DEFAULT_DST}";;
esac esac
log_info "Mapped to: ${destination}"
PATH_CACHE["${source_path}"]="${destination}"
echo "${destination}"
} }
handle_archives() { handle_archives() {
local src="$1" dst="$2" local src="$1" dst="$2"
find "${src}" -type f \( -iname "*.rar" -o -iname "*.zip" -o -iname "*.7z" \) | \ find "${src}" -type f \( -iname "*.rar" -o -iname "*.zip" -o -iname "*.7z" \) | while read -r arch; do
while read -r arch; do
log_info "Extracting ${arch##*/}" log_info "Extracting ${arch##*/}"
case "${arch##*.}" in case "${arch##*.}" in
rar) unrar x -o- "${arch}" "${dst}";; rar) unrar x -o- "${arch}" "${dst}";;
@ -196,30 +259,54 @@ copy_files() {
process_copy() { process_copy() {
local id="$1" hash="$2" src="$3" dst="$4" local id="$1" hash="$2" src="$3" dst="$4"
if [[ ! -d "${src}" ]]; then
log_error "Source directory missing: ${src}"
return 1
fi
if [[ ! -d "${dst}" ]]; then
log_info "Creating destination directory: ${dst}"
mkdir -p "${dst}" || {
log_error "Failed to create directory: ${dst}"
return 1
}
chmod 775 "${dst}"
chown debian-transmission:debian-transmission "${dst}"
fi
if [[ ! -w "${dst}" ]]; then
log_error "No write permissions for: ${dst}"
return 1
fi
if (( DRY_RUN )); then if (( DRY_RUN )); then
log_info "[DRY RUN] Would process torrent ${id}:" log_info "[DRY RUN] Would process torrent ${id}:"
log_info " - Copy files from ${src} to ${dst}" log_info " - Copy files from ${src} to ${dst}"
log_info " - File count: $(find "${src}" -maxdepth 1 -type f | wc -l)"
return return
fi fi
mkdir -p "${dst}"
handle_archives "${src}" "${dst}" handle_archives "${src}" "${dst}"
case "${COPY_MODE}" in case "${COPY_MODE}" in
move) move_files "${dst}" "${src}";; move)
copy) copy_files "${dst}" "${src}";; log_info "Moving files from ${src} to ${dst}"
move_files "${dst}" "${src}"
;;
copy)
log_info "Copying files from ${src} to ${dst}"
copy_files "${dst}" "${src}"
;;
esac esac
if [ $? -eq 0 ]; then
log_info "Transfer completed successfully"
mark_processed "${hash}"
else
log_error "Transfer failed for ${src}"
fi
} }
process_removal() { process_removal() {
local id="$1" local id="$1"
if (( DRY_RUN )); then if (( DRY_RUN )); then
log_info "[DRY RUN] Would remove torrent ${id}" log_info "[DRY RUN] Would remove torrent ${id}"
return return
fi fi
transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \ transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" \ -n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" \
-t "${id}" --remove-and-delete -t "${id}" --remove-and-delete
@ -227,56 +314,65 @@ process_removal() {
main() { main() {
check_dependencies check_dependencies
declare -a REQUIRED_DIRS=(
"${DIR_GAMES_DST}"
"${DIR_APPS_DST}"
"${DIR_MOVIES_DST}"
"${DIR_BOOKS_DST}"
"${DEFAULT_DST}"
)
for dir in "${REQUIRED_DIRS[@]}"; do
if [[ ! -d "${dir}" ]]; then
log_error "Directory missing: ${dir}"
exit 1
fi
if [[ ! -w "${dir}" ]]; then
log_error "Write permission denied: ${dir}"
exit 1
fi
done
init_checksum_db init_checksum_db
if (( CACHE_WARMUP )); then if (( CACHE_WARMUP )); then
warm_cache warm_cache
exit 0 exit 0
fi fi
log_info "Starting processing" log_info "Starting processing"
declare -A warned_dirs=() declare -A warned_dirs=()
transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \ transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -l | \ -n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -l | awk 'NR>1 && $1 ~ /^[0-9]+$/ {print $1}' | while read -r id; do
awk 'NR>1 && $1 ~ /^[0-9]+$/ {print $1}' | \ local info
while read -r id; do info=$(transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
local info=$(transmission-remote "${TRANSMISSION_IP}:${TRANSMISSION_PORT}" \
-n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -t "${id}" -i) -n "${TRANSMISSION_USER}:${TRANSMISSION_PASSWORD}" -t "${id}" -i)
local hash=$(grep "Hash:" <<< "${info}" | awk '{print $2}') local hash
hash=$(grep "Hash:" <<< "${info}" | awk '{print $2}')
# Sanitize numeric values with fallbacks local ratio
local ratio=$(grep "Ratio:" <<< "${info}" | awk '{print $2 == "None" ? 0 : $2}' | tr -cd '0-9.') ratio=$(grep "Ratio:" <<< "${info}" | awk '{print $2 == "None" ? 0 : $2}' | tr -cd '0-9.')
ratio=${ratio:-0} # Handle empty values ratio=${ratio:-0}
local time
local time=$(grep "Seeding Time:" <<< "${info}" | awk '{print $3 == "None" ? 0 : $3}' | tr -cd '0-9.') time=$(grep "Seeding Time:" <<< "${info}" | awk '{print $3 == "None" ? 0 : $3}' | tr -cd '0-9.')
time=${time:-0} # Handle empty values time=${time:-0}
local percent_done
local percent_done=$(grep "Percent Done:" <<< "${info}" | awk '{gsub(/%/, ""); print $3 == "None" ? 0 : $3}') percent_done=$(grep "Percent Done:" <<< "${info}" | awk '{gsub(/%/, ""); print $3 == "None" ? 0 : $3}')
percent_done=${percent_done:-0} # Handle empty values percent_done=${percent_done:-0}
# Extract and translate download location.
local dir=$(grep "Location:" <<< "${info}" | cut -d' ' -f4-) local reported_dir
local dst=$(get_destination "${dir}") reported_dir=$(grep -i "Location:" <<< "${info}" | awk -F": " '{print $2}' | xargs)
local dir
# Initialize warning tracking dir=$(translate_source "${reported_dir}")
log_info "Torrent source directory reported: '${reported_dir}' translated to '${dir}'"
local dst
dst=$(get_destination "${dir}")
[[ -z "${warned_dirs["${dir}"]+x}" ]] && warned_dirs["${dir}"]=0 [[ -z "${warned_dirs["${dir}"]+x}" ]] && warned_dirs["${dir}"]=0
# 1. Handle completed downloads
if (( $(bc <<< "${percent_done} >= 100") )) && ! is_processed "${hash}"; then if (( $(bc <<< "${percent_done} >= 100") )) && ! is_processed "${hash}"; then
log_info "Processing completed torrent ${id} (${percent_done}% done)" log_info "Processing completed torrent ${id} (${percent_done}% done)"
if [[ "${dst}" == "${DEFAULT_DST}" ]] && (( warned_dirs["${dir}"] == 0 )); then if [[ "${dst}" == "${DEFAULT_DST}" ]] && (( warned_dirs["${dir}"] == 0 )); then
log_warn "Using default destination for: ${dir}" log_warn "Using default destination for: ${dir}"
warned_dirs["${dir}"]=1 warned_dirs["${dir}"]=1
fi fi
# Determine check targets
local targets=("${dst}") local targets=("${dst}")
if [[ "${dst}" == "${DIR_MOVIES_DST}" ]]; then case "${dst}" in
targets+=("${STORAGE_DIRS_ARRAY[@]}") "${DIR_MOVIES_DST}") targets+=("${STORAGE_DIRS_ARRAY[@]}");;
fi esac
if ! files_need_processing "${dir}" "${targets[@]}"; then if ! files_need_processing "${dir}" "${targets[@]}"; then
log_info "Skipping copy - files already exist in:" log_info "Skipping copy - files already exist in:"
for target in "${targets[@]}"; do for target in "${targets[@]}"; do
@ -285,18 +381,12 @@ main() {
else else
process_copy "${id}" "${hash}" "${dir}" "${dst}" process_copy "${id}" "${hash}" "${dir}" "${dst}"
fi fi
mark_processed "${hash}"
fi fi
# 2. Handle seeding criteria
if (( $(bc <<< "${ratio} >= ${SEED_RATIO}") )) || (( $(bc <<< "${time} >= ${SEED_TIME}") )); then if (( $(bc <<< "${ratio} >= ${SEED_RATIO}") )) || (( $(bc <<< "${time} >= ${SEED_TIME}") )); then
log_info "Removing torrent ${id} (Ratio: ${ratio}, Time: ${time})" log_info "Removing torrent ${id} (Ratio: ${ratio}, Time: ${time})"
process_removal "${id}" process_removal "${id}"
fi fi
done done
# Final disk checks
check_disk_usage "${DIR_GAMES_DST}" check_disk_usage "${DIR_GAMES_DST}"
check_disk_usage "${DIR_APPS_DST}" check_disk_usage "${DIR_APPS_DST}"
check_disk_usage "${DIR_MOVIES_DST}" check_disk_usage "${DIR_MOVIES_DST}"
@ -304,7 +394,6 @@ main() {
check_disk_usage "${DEFAULT_DST}" check_disk_usage "${DEFAULT_DST}"
} }
# Argument handling
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
case "$1" in case "$1" in
--dry-run) DRY_RUN=1; shift ;; --dry-run) DRY_RUN=1; shift ;;
@ -318,7 +407,6 @@ while [[ $# -gt 0 ]]; do
esac esac
done done
# Execution
if (( INTERACTIVE )); then if (( INTERACTIVE )); then
read -rp "Confirm processing? (y/n) " choice read -rp "Confirm processing? (y/n) " choice
[[ "${choice}" =~ ^[Yy]$ ]] || exit 0 [[ "${choice}" =~ ^[Yy]$ ]] || exit 0