transmission-rss-manager/modules/rss-feed-manager.js

// rss-feed-manager.js - Handles RSS feed fetching, parsing, and torrent management
const fs = require('fs').promises;
const path = require('path');
const fetch = require('node-fetch');
const xml2js = require('xml2js');
const crypto = require('crypto');

class RssFeedManager {
  constructor(config) {
    if (!config) {
      throw new Error('Configuration is required');
    }
    
    this.config = config;
    this.feeds = config.feeds || [];
    this.items = [];
    this.updateIntervalId = null;
    this.updateIntervalMinutes = config.updateIntervalMinutes || 60;
    this.parser = new xml2js.Parser({ explicitArray: false });
    
    // Ensure dataPath is properly defined
    this.dataPath = path.join(__dirname, '..', 'data');
    
    // Maximum items to keep in memory to prevent memory leaks
    this.maxItemsInMemory = config.maxItemsInMemory || 5000;
  }
  
  async start() {
    if (this.updateIntervalId) {
      return;
    }
    
    try {
      // Load existing feeds and items
      await this.loadFeeds();
      await this.loadItems();
      
      // Run update immediately
      await this.updateAllFeeds().catch(error => {
        console.error('Error in initial feed update:', error);
      });
      
      // Then set up interval
      this.updateIntervalId = setInterval(async () => {
        await this.updateAllFeeds().catch(error => {
          console.error('Error in scheduled feed update:', error);
        });
      }, this.updateIntervalMinutes * 60 * 1000);
      
      console.log(`RSS feed manager started, interval: ${this.updateIntervalMinutes} minutes`);
    } catch (error) {
      console.error('Failed to start RSS feed manager:', error);
      throw error;
    }
  }
  
  stop() {
    if (this.updateIntervalId) {
      clearInterval(this.updateIntervalId);
      this.updateIntervalId = null;
      console.log('RSS feed manager stopped');
      return true;
    }
    return false;
  }
  
  async updateAllFeeds() {
    console.log('Updating all RSS feeds...');
    
    const results = [];
    
    // Check if feeds array is valid
    if (!Array.isArray(this.feeds)) {
      console.error('Feeds is not an array:', this.feeds);
      this.feeds = [];
      return results;
    }
    
    for (const feed of this.feeds) {
      if (!feed || !feed.id || !feed.url) {
        console.error('Invalid feed object:', feed);
        continue;
      }
      
      try {
        const result = await this.updateFeed(feed);
        results.push({
          feedId: feed.id,
          success: true,
          newItems: result.newItems
        });
      } catch (error) {
        console.error(`Error updating feed ${feed.id} (${feed.url}):`, error.message);
        results.push({
          feedId: feed.id,
          success: false,
          error: error.message
        });
      }
    }
    
    try {
      // Save updated items and truncate if necessary
      this.trimItemsIfNeeded();
      await this.saveItems();
      await this.saveFeeds();
    } catch (error) {
      console.error('Error saving data after feed update:', error);
    }
    
    console.log('RSS feed update completed');
    return results;
  }
  
  // Trim items to prevent memory bloat
  trimItemsIfNeeded() {
    if (this.items.length > this.maxItemsInMemory) {
      console.log(`Trimming items from ${this.items.length} to ${this.maxItemsInMemory}`);
      
      // Sort by date (newest first) and keep only the newest maxItemsInMemory items
      this.items.sort((a, b) => new Date(b.added) - new Date(a.added));
      this.items = this.items.slice(0, this.maxItemsInMemory);
    }
  }
  
  async updateFeed(feed) {
    if (!feed || !feed.url) {
      throw new Error('Invalid feed configuration');
    }
    
    console.log(`Updating feed: ${feed.name || 'Unnamed'} (${feed.url})`);
    
    try {
      const response = await fetch(feed.url, { 
        timeout: 30000,  // 30 second timeout 
        headers: {
          'User-Agent': 'Transmission-RSS-Manager/2.0.6'
        }
      });
      
      if (!response.ok) {
        throw new Error(`HTTP error ${response.status}: ${response.statusText}`);
      }
      
      const xml = await response.text();
      
      if (!xml || xml.trim() === '') {
        throw new Error('Empty feed content');
      }
      
      const result = await this.parseXml(xml);
      
      if (!result) {
        throw new Error('Failed to parse XML feed');
      }
      
      const rssItems = this.extractItems(result, feed);
      const newItems = this.processNewItems(rssItems, feed);
      
      console.log(`Found ${rssItems.length} items, ${newItems.length} new items in feed: ${feed.name || 'Unnamed'}`);
      
      return {
        totalItems: rssItems.length,
        newItems: newItems.length
      };
    } catch (error) {
      console.error(`Error updating feed ${feed.url}:`, error);
      throw error;
    }
  }
  
  parseXml(xml) {
    if (!xml || typeof xml !== 'string') {
      return Promise.reject(new Error('Invalid XML input'));
    }
    
    return new Promise((resolve, reject) => {
      this.parser.parseString(xml, (error, result) => {
        if (error) {
          reject(error);
        } else {
          resolve(result);
        }
      });
    });
  }
  
  extractItems(parsedXml, feed) {
    if (!parsedXml || !feed) {
      console.error('Invalid parsed XML or feed');
      return [];
    }
    
    try {
      // Handle standard RSS 2.0
      if (parsedXml.rss && parsedXml.rss.channel) {
        const channel = parsedXml.rss.channel;
        
        if (!channel.item) {
          return [];
        }
        
        const items = Array.isArray(channel.item) 
          ? channel.item.filter(Boolean) 
          : (channel.item ? [channel.item] : []);
          
        return items.map(item => this.normalizeRssItem(item, feed));
      }
      
      // Handle Atom
      if (parsedXml.feed && parsedXml.feed.entry) {
        const entries = Array.isArray(parsedXml.feed.entry) 
          ? parsedXml.feed.entry.filter(Boolean) 
          : (parsedXml.feed.entry ? [parsedXml.feed.entry] : []);
          
        return entries.map(entry => this.normalizeAtomItem(entry, feed));
      }
      
      return [];
    } catch (error) {
      console.error('Error extracting items from XML:', error);
      return [];
    }
  }
  
  normalizeRssItem(item, feed) {
    if (!item || !feed) {
      console.error('Invalid RSS item or feed');
      return null;
    }
    
    try {
      // Create a unique ID for the item
      const title = item.title || 'Untitled';
      const pubDate = item.pubDate || '';
      const link = item.link || '';
      const idContent = `${feed.id}:${title}:${pubDate}:${link}`;
      const id = crypto.createHash('md5').update(idContent).digest('hex');
      
      // Extract enclosure (torrent link)
      let torrentLink = link;
      let fileSize = 0;
      
      if (item.enclosure) {
        if (item.enclosure.$) {
          torrentLink = item.enclosure.$.url || torrentLink;
          fileSize = parseInt(item.enclosure.$.length || 0, 10);
        } else if (typeof item.enclosure === 'object') {
          torrentLink = item.enclosure.url || torrentLink;
          fileSize = parseInt(item.enclosure.length || 0, 10);
        }
      }
      
      // Handle custom namespaces (common in torrent feeds)
      let category = '';
      let size = fileSize;
      
      if (item.category) {
        category = Array.isArray(item.category) ? item.category[0] : item.category;
        // Handle if category is an object with a value property
        if (typeof category === 'object' && category._) {
          category = category._;
        }
      }
      
      // Some feeds use torrent:contentLength
      if (item['torrent:contentLength']) {
        const contentLength = parseInt(item['torrent:contentLength'], 10);
        if (!isNaN(contentLength)) {
          size = contentLength;
        }
      }
      
      return {
        id,
        feedId: feed.id,
        title,
        link,
        torrentLink,
        pubDate: pubDate || new Date().toISOString(),
        category: category || '',
        description: item.description || '',
        size: !isNaN(size) ? size : 0,
        downloaded: false,
        ignored: false,
        added: new Date().toISOString()
      };
    } catch (error) {
      console.error('Error normalizing RSS item:', error);
      return null;
    }
  }
  
  normalizeAtomItem(entry, feed) {
    if (!entry || !feed) {
      console.error('Invalid Atom entry or feed');
      return null;
    }
    
    try {
      // Create a unique ID for the item
      const title = entry.title || 'Untitled';
      const updated = entry.updated || '';
      const entryId = entry.id || '';
      const idContent = `${feed.id}:${title}:${updated}:${entryId}`;
      const id = crypto.createHash('md5').update(idContent).digest('hex');
      
      // Extract link
      let link = '';
      let torrentLink = '';
      
      if (entry.link) {
        if (Array.isArray(entry.link)) {
          const links = entry.link.filter(l => l && l.$);
          const alternateLink = links.find(l => l.$ && l.$.rel === 'alternate');
          const torrentTypeLink = links.find(l => l.$ && l.$.type && l.$.type.includes('torrent'));
          
          link = alternateLink && alternateLink.$ && alternateLink.$.href ? 
            alternateLink.$.href : 
            (links[0] && links[0].$ && links[0].$.href ? links[0].$.href : '');
            
          torrentLink = torrentTypeLink && torrentTypeLink.$ && torrentTypeLink.$.href ? 
            torrentTypeLink.$.href : link;
        } else if (entry.link.$ && entry.link.$.href) {
          link = entry.link.$.href;
          torrentLink = link;
        }
      }
      
      // Extract category
      let category = '';
      if (entry.category && entry.category.$ && entry.category.$.term) {
        category = entry.category.$.term;
      }
      
      // Extract content
      let description = '';
      if (entry.summary) {
        description = entry.summary;
      } else if (entry.content) {
        description = entry.content;
      }
      
      return {
        id,
        feedId: feed.id,
        title,
        link,
        torrentLink,
        pubDate: entry.updated || entry.published || new Date().toISOString(),
        category,
        description,
        size: 0, // Atom doesn't typically include file size
        downloaded: false,
        ignored: false,
        added: new Date().toISOString()
      };
    } catch (error) {
      console.error('Error normalizing Atom item:', error);
      return null;
    }
  }
  
  processNewItems(rssItems, feed) {
    if (!Array.isArray(rssItems) || !feed) {
      console.error('Invalid RSS items array or feed');
      return [];
    }
    
    const newItems = [];
    
    // Filter out null items
    const validItems = rssItems.filter(item => item !== null);
    
    for (const item of validItems) {
      // Check if item already exists in our list
      const existingItem = this.items.find(i => i.id === item.id);
      
      if (!existingItem) {
        // Add new item to our list
        this.items.push(item);
        newItems.push(item);
        
        // Auto-download if enabled and matches filters
        if (feed.autoDownload && this.matchesFilters(item, feed.filters)) {
          this.queueItemForDownload(item);
        }
      }
    }
    
    return newItems;
  }
  
  matchesFilters(item, filters) {
    if (!item) return false;
    
    if (!filters || !Array.isArray(filters) || filters.length === 0) {
      return true;
    }
    
    // Check if the item matches any of the filters
    return filters.some(filter => {
      if (!filter) return true;
      
      // Title check
      if (filter.title && typeof item.title === 'string' && 
          !item.title.toLowerCase().includes(filter.title.toLowerCase())) {
        return false;
      }
      
      // Category check
      if (filter.category && typeof item.category === 'string' && 
          !item.category.toLowerCase().includes(filter.category.toLowerCase())) {
        return false;
      }
      
      // Size checks
      if (filter.minSize && typeof item.size === 'number' && item.size < filter.minSize) {
        return false;
      }
      
      if (filter.maxSize && typeof item.size === 'number' && item.size > filter.maxSize) {
        return false;
      }
      
      // All checks passed
      return true;
    });
  }
  
  queueItemForDownload(item) {
    if (!item) return;
    
    // Mark the item as queued for download
    console.log(`Auto-downloading item: ${item.title}`);
    
    // This would be implemented to add to Transmission
    // But we need a reference to the Transmission client
    // In a real implementation, this might publish to a queue that's consumed elsewhere
    item.downloadQueued = true;
  }
  
  async saveItems() {
    try {
      // Ensure data directory exists
      await this.ensureDataDirectory();
      
      // Save items to file
      await fs.writeFile(
        path.join(this.dataPath, 'rss-items.json'),
        JSON.stringify(this.items, null, 2),
        'utf8'
      );
      
      console.log(`Saved ${this.items.length} RSS items to disk`);
      return true;
    } catch (error) {
      console.error('Error saving RSS items:', error);
      return false;
    }
  }
  
  async saveFeeds() {
    try {
      // Ensure data directory exists
      await this.ensureDataDirectory();
      
      // Save feeds to file
      await fs.writeFile(
        path.join(this.dataPath, 'rss-feeds.json'),
        JSON.stringify(this.feeds, null, 2),
        'utf8'
      );
      
      console.log(`Saved ${this.feeds.length} RSS feeds to disk`);
      return true;
    } catch (error) {
      console.error('Error saving RSS feeds:', error);
      return false;
    }
  }
  
  async ensureDataDirectory() {
    try {
      await fs.mkdir(this.dataPath, { recursive: true });
    } catch (error) {
      console.error('Error creating data directory:', error);
      throw error;
    }
  }
  
  async loadItems() {
    try {
      const filePath = path.join(this.dataPath, 'rss-items.json');
      
      // Check if file exists
      try {
        await fs.access(filePath);
      } catch (error) {
        console.log('No saved RSS items found');
        this.items = [];
        return false;
      }
      
      // Load items from file
      const data = await fs.readFile(filePath, 'utf8');
      
      if (!data || data.trim() === '') {
        console.log('Empty RSS items file');
        this.items = [];
        return false;
      }
      
      try {
        const items = JSON.parse(data);
        
        if (Array.isArray(items)) {
          this.items = items;
          console.log(`Loaded ${this.items.length} RSS items from disk`);
          return true;
        } else {
          console.error('RSS items file does not contain an array');
          this.items = [];
          return false;
        }
      } catch (parseError) {
        console.error('Error parsing RSS items JSON:', parseError);
        this.items = [];
        return false;
      }
    } catch (error) {
      console.error('Error loading RSS items:', error);
      this.items = [];
      return false;
    }
  }
  
  async loadFeeds() {
    try {
      const filePath = path.join(this.dataPath, 'rss-feeds.json');
      
      // Check if file exists
      try {
        await fs.access(filePath);
      } catch (error) {
        console.log('No saved RSS feeds found, using config feeds');
        return false;
      }
      
      // Load feeds from file
      const data = await fs.readFile(filePath, 'utf8');
      
      if (!data || data.trim() === '') {
        console.log('Empty RSS feeds file, using config feeds');
        return false;
      }
      
      try {
        const feeds = JSON.parse(data);
        
        if (Array.isArray(feeds)) {
          this.feeds = feeds;
          console.log(`Loaded ${this.feeds.length} RSS feeds from disk`);
          return true;
        } else {
          console.error('RSS feeds file does not contain an array');
          return false;
        }
      } catch (parseError) {
        console.error('Error parsing RSS feeds JSON:', parseError);
        return false;
      }
    } catch (error) {
      console.error('Error loading RSS feeds:', error);
      return false;
    }
  }
  
  // Public API methods
  
  getAllFeeds() {
    return Array.isArray(this.feeds) ? this.feeds : [];
  }
  
  addFeed(feedData) {
    if (!feedData || !feedData.url) {
      throw new Error('Feed URL is required');
    }
    
    // Generate an ID for the feed
    const id = crypto.randomBytes(8).toString('hex');
    
    const newFeed = {
      id,
      name: feedData.name || 'Unnamed Feed',
      url: feedData.url,
      autoDownload: !!feedData.autoDownload,
      filters: Array.isArray(feedData.filters) ? feedData.filters : [],
      added: new Date().toISOString()
    };
    
    if (!Array.isArray(this.feeds)) {
      this.feeds = [];
    }
    
    this.feeds.push(newFeed);
    
    // Save the updated feeds
    this.saveFeeds().catch(err => {
      console.error('Error saving feeds after adding new feed:', err);
    });
    
    console.log(`Added new feed: ${newFeed.name} (${newFeed.url})`);
    
    return newFeed;
  }
  
  updateFeedConfig(feedId, updates) {
    if (!feedId || !updates) {
      return false;
    }
    
    if (!Array.isArray(this.feeds)) {
      console.error('Feeds is not an array');
      return false;
    }
    
    const feedIndex = this.feeds.findIndex(f => f && f.id === feedId);
    
    if (feedIndex === -1) {
      console.error(`Feed with ID ${feedId} not found`);
      return false;
    }
    
    // Update the feed, preserving the id and added date
    this.feeds[feedIndex] = {
      ...this.feeds[feedIndex],
      ...updates,
      id: feedId,
      added: this.feeds[feedIndex].added
    };
    
    // Save the updated feeds
    this.saveFeeds().catch(err => {
      console.error('Error saving feeds after updating feed:', err);
    });
    
    console.log(`Updated feed: ${this.feeds[feedIndex].name}`);
    
    return true;
  }
  
  removeFeed(feedId) {
    if (!feedId || !Array.isArray(this.feeds)) {
      return false;
    }
    
    const initialLength = this.feeds.length;
    this.feeds = this.feeds.filter(f => f && f.id !== feedId);
    
    if (this.feeds.length !== initialLength) {
      // Save the updated feeds
      this.saveFeeds().catch(err => {
        console.error('Error saving feeds after removing feed:', err);
      });
      
      return true;
    }
    
    return false;
  }
  
  getAllItems() {
    return Array.isArray(this.items) ? this.items : [];
  }
  
  getUndownloadedItems() {
    if (!Array.isArray(this.items)) {
      return [];
    }
    return this.items.filter(item => item && !item.downloaded && !item.ignored);
  }
  
  filterItems(filters) {
    if (!filters || !Array.isArray(this.items)) {
      return [];
    }
    return this.items.filter(item => item && this.matchesFilters(item, [filters]));
  }
  
  async downloadItem(item, transmissionClient) {
    if (!item || !item.torrentLink) {
      return {
        success: false,
        message: 'Invalid item or missing torrent link'
      };
    }
    
    if (!transmissionClient) {
      return {
        success: false,
        message: 'Transmission client not available'
      };
    }
    
    return new Promise((resolve) => {
      transmissionClient.addUrl(item.torrentLink, async (err, result) => {
        if (err) {
          console.error(`Error adding torrent for ${item.title}:`, err);
          resolve({
            success: false,
            message: `Error adding torrent: ${err.message}`,
            result: null
          });
          return;
        }
        
        // Mark the item as downloaded
        item.downloaded = true;
        item.downloadDate = new Date().toISOString();
        
        // Save the updated items
        try {
          await this.saveItems();
        } catch (err) {
          console.error('Error saving items after download:', err);
        }
        
        console.log(`Successfully added torrent for item: ${item.title}`);
        
        resolve({
          success: true,
          message: 'Torrent added successfully',
          result
        });
      });
    });
  }
}

module.exports = RssFeedManager;