torrent/usr/local/lib/torrent-mover/file_operations.sh

#!/bin/bash
# File operation functions for torrent-mover

# init_checksum_db: Initializes the checksum database.
init_checksum_db() {
    mkdir -p "$(dirname "${CHECKSUM_DB}")"
    touch "${CHECKSUM_DB}" || { log_error "Could not create ${CHECKSUM_DB}"; exit 1; }
    chmod 600 "${CHECKSUM_DB}"
}

# record_checksums: Generates checksums for files in given directories.
record_checksums() {
    log_info "Generating checksums with ${PARALLEL_THREADS:-$(nproc)} threads"
    find "$@" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -print0 | \
        parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | sort > "${CHECKSUM_DB}.tmp"
    mv "${CHECKSUM_DB}.tmp" "${CHECKSUM_DB}"
}

# generate_checksums: Common function to generate checksums efficiently
generate_checksums() {
    local dir="$1"
    local cache_file="${CHECKSUM_DB}.$(echo "$dir" | md5sum | cut -d' ' -f1)"
    local last_modified_file
    
    # Skip if directory doesn't exist
    if [[ ! -d "${dir}" ]]; then
        return 1
    fi
    
    # Get the most recently modified file in the directory
    last_modified_file=$(find "${dir}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -exec stat -c "%Y %n" {} \; | sort -nr | head -n1 | cut -d' ' -f2-)
    
    # If cache exists and no files were modified since last cache, use cache
    if [[ -f "${cache_file}" ]] && [[ -n "${last_modified_file}" ]]; then
        local cache_time file_time
        cache_time=$(stat -c "%Y" "${cache_file}")
        file_time=$(stat -c "%Y" "${last_modified_file}")
        
        if (( cache_time >= file_time )); then
            log_debug "Using cached checksums for ${dir}"
            cat "${cache_file}"
            return 0
        fi
    fi
    
    # Generate new checksums with parallel processing
    log_debug "Generating fresh checksums for ${dir}"
    find "${dir}" -type f ! \( -iname "*.nfo" -o -iname "*.sfv" \) -print0 | \
        parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} md5sum | sort | tee "${cache_file}"
    
    return 0
}

# file_metadata: Returns an md5 hash for file metadata.
file_metadata() {
    generate_checksums "$1" | awk '{print $1}'
}

# files_need_processing: Checks if the source files need processing.
files_need_processing() {
    local src="$1"
    shift
    local targets=("$@")
    
    if [[ ! -d "${src}" ]]; then
        log_warn "Source directory missing: ${src}"
        return 1
    fi

    log_info "=== FILE VERIFICATION DEBUG START ==="
    log_info "Source directory: ${src}"
    log_info "Verification targets: ${targets[*]}"

    local empty_target_found=0
    for target in "${targets[@]}"; do
        if [[ ! -d "${target}" ]]; then
            log_info "Target missing: ${target}"
            empty_target_found=1
            continue
        fi

        local file_count
        file_count=$(find "${target}" -mindepth 1 -maxdepth 1 -print | wc -l)
        log_debug "File count for target ${target}: ${file_count}"
        if [[ "${file_count}" -eq 0 ]]; then
            log_info "Empty target directory: ${target}"
            empty_target_found=1
        else
            log_info "Target contains ${file_count} items: ${target}"
            log_info "First 5 items:"
            find "${target}" -mindepth 1 -maxdepth 1 | head -n 5 | while read -r item; do
                log_info "    - ${item##*/}"
            done
        fi
    done

    if [[ "${empty_target_found}" -eq 1 ]]; then
        log_info "Empty target detected - processing needed"
        log_info "=== FILE VERIFICATION DEBUG END ==="
        return 0
    fi

    log_info "Generating source checksums..."
    local src_checksums
    src_checksums=$(generate_checksums "${src}")
    log_info "First 5 source checksums:"
    echo "${src_checksums}" | head -n 5 | while read -r line; do
        log_info "    ${line}"
    done

    local match_found=0
    for target in "${targets[@]}"; do
        log_info "Checking against target: ${target}"
        log_info "Generating target checksums..."
        local target_checksums
        target_checksums=$(generate_checksums "${target}")
        log_info "First 5 target checksums:"
        echo "${target_checksums}" | head -n 5 | while read -r line; do
            log_info "    ${line}"
        done

        if diff <(echo "${src_checksums}") <(echo "${target_checksums}") >/dev/null; then
            log_info "Exact checksum match found in: ${target}"
            match_found=1
            break
        else
            log_info "No match in: ${target}"
        fi
    done

    log_info "=== FILE VERIFICATION DEBUG END ==="
    [[ "${match_found}" -eq 1 ]] && return 1 || return 0
}

# warm_cache: Pre-calculates checksums for storage directories.
warm_cache() {
    log_info "Starting cache warmup for Movies..."
    local targets=("${DIR_MOVIES_DST}" "${STORAGE_DIRS_ARRAY[@]}")
    record_checksums "${targets[@]}"
    log_info "Cache warmup completed. Checksums stored in ${CHECKSUM_DB}"
}

# is_processed: Checks if the torrent (by hash) has already been processed.
is_processed() {
    grep -q "^${1}$" "${PROCESSED_LOG}" 2>/dev/null
}

# mark_processed: Records a processed torrent.
mark_processed() {
    echo "${1}" >> "${PROCESSED_LOG}"
}

# move_files: Moves files using parallel processing if enabled.
move_files() {
    if (( PARALLEL_PROCESSING )); then
        retry_command "parallel -j ${PARALLEL_THREADS:-$(nproc)} mv {} \"${1}\" ::: \"${2}\"/*" 3 15
    else
        retry_command "mv \"${2}\"/* \"${1}\"" 3 15
    fi
}

# copy_files: Copies files using parallel processing if enabled.
copy_files() {
    if (( PARALLEL_PROCESSING )); then
        retry_command "parallel -j ${PARALLEL_THREADS:-$(nproc)} cp -r {} \"${1}\" ::: \"${2}\"/*" 3 15
    else
        retry_command "cp -r \"${2}\"/* \"${1}\"" 3 15
    fi
}

# check_seeding_status: Check if torrent is still seeding
check_seeding_status() {
    local id="$1"
    local status
    
    # Get torrent status from transmission
    status=$(transmission-remote --auth "${TRANSMISSION_USER}:${TRANSMISSION_PASS}" --torrent "${id}" --info | grep "State:" | awk '{print $2}')
    
    # Return 0 if seeding (meaning it's active), 1 if it's not seeding
    if [[ "$status" == "Seeding" ]]; then
        log_info "Torrent ${id} is actively seeding"
        return 0
    else
        log_info "Torrent ${id} is not seeding (status: ${status})"
        return 1
    fi
}

# safe_move_files: Either move files or create hardlinks depending on seeding status
safe_move_files() {
    local dst="$1" src="$2" id="$3"
    
    # If torrent is seeding, use hardlinks instead of moving
    if check_seeding_status "${id}"; then
        log_info "Using hardlinks for seeding torrent ${id}"
        if (( PARALLEL_PROCESSING )); then
            # Using cp with --link to create hardlinks instead of copying
            retry_command "find \"${src}\" -type f -print0 | parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} cp --link {} \"${dst}/\" 2>/dev/null || cp {} \"${dst}/\"" 3 15
            # Handle directories separately - we need to create them first
            retry_command "find \"${src}\" -type d -print0 | parallel -0 -j ${PARALLEL_THREADS:-$(nproc)} mkdir -p \"${dst}/{}\"" 3 15
        else
            # Non-parallel hardlink creation
            retry_command "find \"${src}\" -type f -exec cp --link {} \"${dst}/\" \; 2>/dev/null || cp {} \"${dst}/\"" 3 15
            retry_command "find \"${src}\" -type d -exec mkdir -p \"${dst}/{}\" \;" 3 15
        fi
    else
        # If not seeding, proceed with normal move operation
        move_files "${dst}" "${src}"
    fi
}

# process_copy: Validates directories, then copies/moves files from source to destination.
# Optionally verifies integrity after transfer if CHECK_TRANSFER_INTEGRITY is "true".
process_copy() {
    local id="$1" hash="$2" src="$3" dst="$4"
    local operation_result=0
    
    if [[ ! -d "${src}" ]]; then
        log_error "Source directory missing: ${src}"
        return 1
    fi
    
    # Create destination with proper error handling
    if [[ ! -d "${dst}" ]]; then
        log_info "Creating destination directory: ${dst}"
        if ! mkdir -p "${dst}"; then
            log_error "Failed to create directory: ${dst}"
            return 1
        fi
        chmod 775 "${dst}"
        chown ${TORRENT_USER:-debian-transmission}:${TORRENT_GROUP:-debian-transmission} "${dst}"
    fi
    
    if [[ ! -w "${dst}" ]]; then
        log_error "No write permissions for: ${dst}"
        return 1
    fi
    
    if (( DRY_RUN )); then
        log_info "[DRY RUN] Would process torrent ${id}:"
        log_info "  - Copy files from ${src} to ${dst}"
        log_info "  - File count: $(find "${src}" -maxdepth 1 -type f | wc -l)"
        return 0
    fi
    
    # Extract archives first
    if ! handle_archives "${src}" "${dst}"; then
        log_warn "Archive extraction had issues for ${src}, continuing with regular files"
    fi
    
    # Process files atomically
    case "${COPY_MODE}" in
        move)
            log_info "Moving files from ${src} to ${dst}"
            safe_move_files "${dst}" "${src}" "${id}"
            operation_result=$?
            ;;
        copy)
            log_info "Copying files from ${src} to ${dst}"
            copy_files "${dst}" "${src}"
            operation_result=$?
            ;;
    esac
    
    if [[ ${operation_result} -eq 0 ]]; then
        if [[ "${CHECK_TRANSFER_INTEGRITY}" == "true" ]]; then
            log_info "Verifying integrity of transferred files..."
            local src_checksum target_checksum
            src_checksum=$(generate_checksums "${src}")
            target_checksum=$(generate_checksums "${dst}")
            
            if diff <(echo "${src_checksum}") <(echo "${target_checksum}") >/dev/null; then
                log_info "Integrity check passed."
            else
                log_error "Integrity check FAILED for ${src}"
                return 1
            fi
        fi
        
        log_info "Transfer completed successfully"
        mark_processed "${hash}"
    else
        log_error "Transfer failed for ${src}"
        return 1
    fi
    
    return 0
}