#!/bin/bash # File operation functions for torrent-mover # init_checksum_db: Initializes the checksum database. init_checksum_db() { mkdir -p "$(dirname "${CHECKSUM_DB}")" touch "${CHECKSUM_DB}" || { log_error "Could not create ${CHECKSUM_DB}"; exit 1; } chmod 600 "${CHECKSUM_DB}" } # record_checksums: Generates checksums for files in given directories. record_checksums() { log_info "Generating checksums with ${PARALLEL_THREADS:-$(nproc)} threads" find "$@" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -print0 | \ parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | sort > "${CHECKSUM_DB}.tmp" mv "${CHECKSUM_DB}.tmp" "${CHECKSUM_DB}" } # generate_checksums: Common function to generate checksums efficiently generate_checksums() { local dir="$1" local cache_file="${CHECKSUM_DB}.$(echo "$dir" | md5sum | cut -d' ' -f1)" local last_modified_file # Skip if directory doesn't exist if [[ ! -d "${dir}" ]]; then return 1 fi # Get the most recently modified file in the directory last_modified_file=$(find "${dir}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec stat -c "%Y %n" {} \; | sort -nr | head -n1 | cut -d' ' -f2-) # If cache exists and no files were modified since last cache, use cache if [[ -f "${cache_file}" ]] && [[ -n "${last_modified_file}" ]]; then local cache_time file_time cache_time=$(stat -c "%Y" "${cache_file}") file_time=$(stat -c "%Y" "${last_modified_file}") if (( cache_time >= file_time )); then log_debug "Using cached checksums for ${dir}" cat "${cache_file}" return 0 fi fi # Generate new checksums with parallel processing log_debug "Generating fresh checksums for ${dir}" find "${dir}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -print0 | \ parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | sort | tee "${cache_file}" return 0 } # file_metadata: Returns an md5 hash for file metadata. file_metadata() { generate_checksums "$1" | awk '{print $1}' } # files_need_processing: Checks if the source files need processing. files_need_processing() { local src="$1" shift local targets=("$@") if [[ ! -d "${src}" ]]; then log_warn "Source directory missing: ${src}" return 1 fi log_info "=== FILE VERIFICATION DEBUG START ===" log_info "Source directory: ${src}" log_info "Verification targets: ${targets[*]}" local empty_target_found=0 for target in "${targets[@]}"; do if [[ ! -d "${target}" ]]; then log_info "Target missing: ${target}" empty_target_found=1 continue fi local file_count file_count=$(find "${target}" -mindepth 1 -maxdepth 1 -print | wc -l) log_debug "File count for target ${target}: ${file_count}" if [[ "${file_count}" -eq 0 ]]; then log_info "Empty target directory: ${target}" empty_target_found=1 else log_info "Target contains ${file_count} items: ${target}" log_info "First 5 items:" find "${target}" -mindepth 1 -maxdepth 1 | head -n 5 | while read -r item; do log_info " - ${item##*/}" done fi done if [[ "${empty_target_found}" -eq 1 ]]; then log_info "Empty target detected - processing needed" log_info "=== FILE VERIFICATION DEBUG END ===" return 0 fi log_info "Generating source checksums..." local src_checksums src_checksums=$(generate_checksums "${src}") log_info "First 5 source checksums:" echo "${src_checksums}" | head -n 5 | while read -r line; do log_info " ${line}" done local match_found=0 for target in "${targets[@]}"; do log_info "Checking against target: ${target}" log_info "Generating target checksums..." local target_checksums target_checksums=$(generate_checksums "${target}") log_info "First 5 target checksums:" echo "${target_checksums}" | head -n 5 | while read -r line; do log_info " ${line}" done if diff <(echo "${src_checksums}") <(echo "${target_checksums}") >/dev/null; then log_info "Exact checksum match found in: ${target}" match_found=1 break else log_info "No match in: ${target}" fi done log_info "=== FILE VERIFICATION DEBUG END ===" [[ "${match_found}" -eq 1 ]] && return 1 || return 0 } # warm_cache: Pre-calculates checksums for storage directories. warm_cache() { log_info "Starting cache warmup for Movies..." local targets=("${DIR_MOVIES_DST}" "${STORAGE_DIRS_ARRAY[@]}") record_checksums "${targets[@]}" log_info "Cache warmup completed. Checksums stored in ${CHECKSUM_DB}" } # is_processed: Checks if the torrent (by hash) has already been processed. is_processed() { grep -q "^${1}$" "${PROCESSED_LOG}" 2>/dev/null } # mark_processed: Records a processed torrent. mark_processed() { echo "${1}" >> "${PROCESSED_LOG}" } # move_files: Moves files using parallel processing if enabled. move_files() { if (( PARALLEL_PROCESSING )); then retry_command "parallel -j ${PARALLEL_THREADS:-$(nproc)} mv {} \"${1}\" ::: \"${2}\"/*" 3 15 else retry_command "mv \"${2}\"/* \"${1}\"" 3 15 fi } # copy_files: Copies files using parallel processing if enabled. copy_files() { if (( PARALLEL_PROCESSING )); then retry_command "parallel -j ${PARALLEL_THREADS:-$(nproc)} cp -r {} \"${1}\" ::: \"${2}\"/*" 3 15 else retry_command "cp -r \"${2}\"/* \"${1}\"" 3 15 fi } # check_seeding_status: Check if torrent is still seeding check_seeding_status() { local id="$1" local status # Get torrent status from transmission status=$(transmission-remote --auth "${TRANSMISSION_USER}:${TRANSMISSION_PASS}" --torrent "${id}" --info | grep "State:" | awk '{print $2}') # Return 0 if seeding (meaning it's active), 1 if it's not seeding if [[ "$status" == "Seeding" ]]; then log_info "Torrent ${id} is actively seeding" return 0 else log_info "Torrent ${id} is not seeding (status: ${status})" return 1 fi } # safe_move_files: Either move files or create hardlinks depending on seeding status safe_move_files() { local dst="$1" src="$2" id="$3" # If torrent is seeding, use hardlinks instead of moving if check_seeding_status "${id}"; then log_info "Using hardlinks for seeding torrent ${id}" if (( PARALLEL_PROCESSING )); then # Using cp with --link to create hardlinks instead of copying retry_command "find \"${src}\" -type f -print0 | parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} cp --link {} \"${dst}/\" 2>/dev/null || cp {} \"${dst}/\"" 3 15 # Handle directories separately - we need to create them first retry_command "find \"${src}\" -type d -print0 | parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} mkdir -p \"${dst}/{}\"" 3 15 else # Non-parallel hardlink creation retry_command "find \"${src}\" -type f -exec cp --link {} \"${dst}/\" \; 2>/dev/null || cp {} \"${dst}/\"" 3 15 retry_command "find \"${src}\" -type d -exec mkdir -p \"${dst}/{}\" \;" 3 15 fi else # If not seeding, proceed with normal move operation move_files "${dst}" "${src}" fi } # process_copy: Validates directories, then copies/moves files from source to destination. # Optionally verifies integrity after transfer if CHECK_TRANSFER_INTEGRITY is "true". process_copy() { local id="$1" hash="$2" src="$3" dst="$4" local operation_result=0 if [[ ! -d "${src}" ]]; then log_error "Source directory missing: ${src}" return 1 fi # Create destination with proper error handling if [[ ! -d "${dst}" ]]; then log_info "Creating destination directory: ${dst}" if ! mkdir -p "${dst}"; then log_error "Failed to create directory: ${dst}" return 1 fi chmod 775 "${dst}" chown ${TORRENT_USER:-debian-transmission}:${TORRENT_GROUP:-debian-transmission} "${dst}" fi if [[ ! -w "${dst}" ]]; then log_error "No write permissions for: ${dst}" return 1 fi if (( DRY_RUN )); then log_info "[DRY RUN] Would process torrent ${id}:" log_info " - Copy files from ${src} to ${dst}" log_info " - File count: $(find "${src}" -maxdepth 1 -type f | wc -l)" return 0 fi # Extract archives first if ! handle_archives "${src}" "${dst}"; then log_warn "Archive extraction had issues for ${src}, continuing with regular files" fi # Process files atomically case "${COPY_MODE}" in move) log_info "Moving files from ${src} to ${dst}" safe_move_files "${dst}" "${src}" "${id}" operation_result=$? ;; copy) log_info "Copying files from ${src} to ${dst}" copy_files "${dst}" "${src}" operation_result=$? ;; esac if [[ ${operation_result} -eq 0 ]]; then if [[ "${CHECK_TRANSFER_INTEGRITY}" == "true" ]]; then log_info "Verifying integrity of transferred files..." local src_checksum target_checksum src_checksum=$(generate_checksums "${src}") target_checksum=$(generate_checksums "${dst}") if diff <(echo "${src_checksum}") <(echo "${target_checksum}") >/dev/null; then log_info "Integrity check passed." else log_error "Integrity check FAILED for ${src}" return 1 fi fi log_info "Transfer completed successfully" mark_processed "${hash}" else log_error "Transfer failed for ${src}" return 1 fi return 0 }