// rss-feed-manager.js - Handles RSS feed fetching, parsing, and torrent management const fs = require('fs').promises; const path = require('path'); const fetch = require('node-fetch'); const xml2js = require('xml2js'); const crypto = require('crypto'); class RssFeedManager { constructor(config) { if (!config) { throw new Error('Configuration is required'); } this.config = config; this.feeds = config.feeds || []; this.items = []; this.updateIntervalId = null; this.updateIntervalMinutes = config.updateIntervalMinutes || 60; this.parser = new xml2js.Parser({ explicitArray: false }); // Ensure dataPath is properly defined this.dataPath = path.join(__dirname, '..', 'data'); // Maximum items to keep in memory to prevent memory leaks this.maxItemsInMemory = config.maxItemsInMemory || 5000; } async start() { if (this.updateIntervalId) { return; } try { // Load existing feeds and items await this.loadFeeds(); await this.loadItems(); // Run update immediately await this.updateAllFeeds().catch(error => { console.error('Error in initial feed update:', error); }); // Then set up interval this.updateIntervalId = setInterval(async () => { await this.updateAllFeeds().catch(error => { console.error('Error in scheduled feed update:', error); }); }, this.updateIntervalMinutes * 60 * 1000); console.log(`RSS feed manager started, interval: ${this.updateIntervalMinutes} minutes`); } catch (error) { console.error('Failed to start RSS feed manager:', error); throw error; } } stop() { if (this.updateIntervalId) { clearInterval(this.updateIntervalId); this.updateIntervalId = null; console.log('RSS feed manager stopped'); return true; } return false; } async updateAllFeeds() { console.log('Updating all RSS feeds...'); const results = []; // Check if feeds array is valid if (!Array.isArray(this.feeds)) { console.error('Feeds is not an array:', this.feeds); this.feeds = []; return results; } for (const feed of this.feeds) { if (!feed || !feed.id || !feed.url) { console.error('Invalid feed object:', feed); continue; } try { const result = await this.updateFeed(feed); results.push({ feedId: feed.id, success: true, newItems: result.newItems }); } catch (error) { console.error(`Error updating feed ${feed.id} (${feed.url}):`, error.message); results.push({ feedId: feed.id, success: false, error: error.message }); } } try { // Save updated items and truncate if necessary this.trimItemsIfNeeded(); await this.saveItems(); await this.saveFeeds(); } catch (error) { console.error('Error saving data after feed update:', error); } console.log('RSS feed update completed'); return results; } // Trim items to prevent memory bloat trimItemsIfNeeded() { if (this.items.length > this.maxItemsInMemory) { console.log(`Trimming items from ${this.items.length} to ${this.maxItemsInMemory}`); // Sort by date (newest first) and keep only the newest maxItemsInMemory items this.items.sort((a, b) => new Date(b.added) - new Date(a.added)); this.items = this.items.slice(0, this.maxItemsInMemory); } } async updateFeed(feed) { if (!feed || !feed.url) { throw new Error('Invalid feed configuration'); } console.log(`Updating feed: ${feed.name || 'Unnamed'} (${feed.url})`); try { const response = await fetch(feed.url, { timeout: 30000, // 30 second timeout headers: { 'User-Agent': 'Transmission-RSS-Manager/1.2.0' } }); if (!response.ok) { throw new Error(`HTTP error ${response.status}: ${response.statusText}`); } const xml = await response.text(); if (!xml || xml.trim() === '') { throw new Error('Empty feed content'); } const result = await this.parseXml(xml); if (!result) { throw new Error('Failed to parse XML feed'); } const rssItems = this.extractItems(result, feed); const newItems = this.processNewItems(rssItems, feed); console.log(`Found ${rssItems.length} items, ${newItems.length} new items in feed: ${feed.name || 'Unnamed'}`); return { totalItems: rssItems.length, newItems: newItems.length }; } catch (error) { console.error(`Error updating feed ${feed.url}:`, error); throw error; } } parseXml(xml) { if (!xml || typeof xml !== 'string') { return Promise.reject(new Error('Invalid XML input')); } return new Promise((resolve, reject) => { this.parser.parseString(xml, (error, result) => { if (error) { reject(error); } else { resolve(result); } }); }); } extractItems(parsedXml, feed) { if (!parsedXml || !feed) { console.error('Invalid parsed XML or feed'); return []; } try { // Handle standard RSS 2.0 if (parsedXml.rss && parsedXml.rss.channel) { const channel = parsedXml.rss.channel; if (!channel.item) { return []; } const items = Array.isArray(channel.item) ? channel.item.filter(Boolean) : (channel.item ? [channel.item] : []); return items.map(item => this.normalizeRssItem(item, feed)); } // Handle Atom if (parsedXml.feed && parsedXml.feed.entry) { const entries = Array.isArray(parsedXml.feed.entry) ? parsedXml.feed.entry.filter(Boolean) : (parsedXml.feed.entry ? [parsedXml.feed.entry] : []); return entries.map(entry => this.normalizeAtomItem(entry, feed)); } return []; } catch (error) { console.error('Error extracting items from XML:', error); return []; } } normalizeRssItem(item, feed) { if (!item || !feed) { console.error('Invalid RSS item or feed'); return null; } try { // Create a unique ID for the item const title = item.title || 'Untitled'; const pubDate = item.pubDate || ''; const link = item.link || ''; const idContent = `${feed.id}:${title}:${pubDate}:${link}`; const id = crypto.createHash('md5').update(idContent).digest('hex'); // Extract enclosure (torrent link) let torrentLink = link; let fileSize = 0; if (item.enclosure) { if (item.enclosure.$) { torrentLink = item.enclosure.$.url || torrentLink; fileSize = parseInt(item.enclosure.$.length || 0, 10); } else if (typeof item.enclosure === 'object') { torrentLink = item.enclosure.url || torrentLink; fileSize = parseInt(item.enclosure.length || 0, 10); } } // Handle custom namespaces (common in torrent feeds) let category = ''; let size = fileSize; if (item.category) { category = Array.isArray(item.category) ? item.category[0] : item.category; // Handle if category is an object with a value property if (typeof category === 'object' && category._) { category = category._; } } // Some feeds use torrent:contentLength if (item['torrent:contentLength']) { const contentLength = parseInt(item['torrent:contentLength'], 10); if (!isNaN(contentLength)) { size = contentLength; } } return { id, feedId: feed.id, title, link, torrentLink, pubDate: pubDate || new Date().toISOString(), category: category || '', description: item.description || '', size: !isNaN(size) ? size : 0, downloaded: false, ignored: false, added: new Date().toISOString() }; } catch (error) { console.error('Error normalizing RSS item:', error); return null; } } normalizeAtomItem(entry, feed) { if (!entry || !feed) { console.error('Invalid Atom entry or feed'); return null; } try { // Create a unique ID for the item const title = entry.title || 'Untitled'; const updated = entry.updated || ''; const entryId = entry.id || ''; const idContent = `${feed.id}:${title}:${updated}:${entryId}`; const id = crypto.createHash('md5').update(idContent).digest('hex'); // Extract link let link = ''; let torrentLink = ''; if (entry.link) { if (Array.isArray(entry.link)) { const links = entry.link.filter(l => l && l.$); const alternateLink = links.find(l => l.$ && l.$.rel === 'alternate'); const torrentTypeLink = links.find(l => l.$ && l.$.type && l.$.type.includes('torrent')); link = alternateLink && alternateLink.$ && alternateLink.$.href ? alternateLink.$.href : (links[0] && links[0].$ && links[0].$.href ? links[0].$.href : ''); torrentLink = torrentTypeLink && torrentTypeLink.$ && torrentTypeLink.$.href ? torrentTypeLink.$.href : link; } else if (entry.link.$ && entry.link.$.href) { link = entry.link.$.href; torrentLink = link; } } // Extract category let category = ''; if (entry.category && entry.category.$ && entry.category.$.term) { category = entry.category.$.term; } // Extract content let description = ''; if (entry.summary) { description = entry.summary; } else if (entry.content) { description = entry.content; } return { id, feedId: feed.id, title, link, torrentLink, pubDate: entry.updated || entry.published || new Date().toISOString(), category, description, size: 0, // Atom doesn't typically include file size downloaded: false, ignored: false, added: new Date().toISOString() }; } catch (error) { console.error('Error normalizing Atom item:', error); return null; } } processNewItems(rssItems, feed) { if (!Array.isArray(rssItems) || !feed) { console.error('Invalid RSS items array or feed'); return []; } const newItems = []; // Filter out null items const validItems = rssItems.filter(item => item !== null); for (const item of validItems) { // Check if item already exists in our list const existingItem = this.items.find(i => i.id === item.id); if (!existingItem) { // Add new item to our list this.items.push(item); newItems.push(item); // Auto-download if enabled and matches filters if (feed.autoDownload && this.matchesFilters(item, feed.filters)) { this.queueItemForDownload(item); } } } return newItems; } matchesFilters(item, filters) { if (!item) return false; if (!filters || !Array.isArray(filters) || filters.length === 0) { return true; } // Check if the item matches any of the filters return filters.some(filter => { if (!filter) return true; // Title check if (filter.title && typeof item.title === 'string' && !item.title.toLowerCase().includes(filter.title.toLowerCase())) { return false; } // Category check if (filter.category && typeof item.category === 'string' && !item.category.toLowerCase().includes(filter.category.toLowerCase())) { return false; } // Size checks if (filter.minSize && typeof item.size === 'number' && item.size < filter.minSize) { return false; } if (filter.maxSize && typeof item.size === 'number' && item.size > filter.maxSize) { return false; } // All checks passed return true; }); } queueItemForDownload(item) { if (!item) return; // Mark the item as queued for download console.log(`Auto-downloading item: ${item.title}`); // This would be implemented to add to Transmission // But we need a reference to the Transmission client // In a real implementation, this might publish to a queue that's consumed elsewhere item.downloadQueued = true; } async saveItems() { try { // Ensure data directory exists await this.ensureDataDirectory(); // Save items to file await fs.writeFile( path.join(this.dataPath, 'rss-items.json'), JSON.stringify(this.items, null, 2), 'utf8' ); console.log(`Saved ${this.items.length} RSS items to disk`); return true; } catch (error) { console.error('Error saving RSS items:', error); return false; } } async saveFeeds() { try { // Ensure data directory exists await this.ensureDataDirectory(); // Save feeds to file await fs.writeFile( path.join(this.dataPath, 'rss-feeds.json'), JSON.stringify(this.feeds, null, 2), 'utf8' ); console.log(`Saved ${this.feeds.length} RSS feeds to disk`); return true; } catch (error) { console.error('Error saving RSS feeds:', error); return false; } } async ensureDataDirectory() { try { await fs.mkdir(this.dataPath, { recursive: true }); } catch (error) { console.error('Error creating data directory:', error); throw error; } } async loadItems() { try { const filePath = path.join(this.dataPath, 'rss-items.json'); // Check if file exists try { await fs.access(filePath); } catch (error) { console.log('No saved RSS items found'); this.items = []; return false; } // Load items from file const data = await fs.readFile(filePath, 'utf8'); if (!data || data.trim() === '') { console.log('Empty RSS items file'); this.items = []; return false; } try { const items = JSON.parse(data); if (Array.isArray(items)) { this.items = items; console.log(`Loaded ${this.items.length} RSS items from disk`); return true; } else { console.error('RSS items file does not contain an array'); this.items = []; return false; } } catch (parseError) { console.error('Error parsing RSS items JSON:', parseError); this.items = []; return false; } } catch (error) { console.error('Error loading RSS items:', error); this.items = []; return false; } } async loadFeeds() { try { const filePath = path.join(this.dataPath, 'rss-feeds.json'); // Check if file exists try { await fs.access(filePath); } catch (error) { console.log('No saved RSS feeds found, using config feeds'); return false; } // Load feeds from file const data = await fs.readFile(filePath, 'utf8'); if (!data || data.trim() === '') { console.log('Empty RSS feeds file, using config feeds'); return false; } try { const feeds = JSON.parse(data); if (Array.isArray(feeds)) { this.feeds = feeds; console.log(`Loaded ${this.feeds.length} RSS feeds from disk`); return true; } else { console.error('RSS feeds file does not contain an array'); return false; } } catch (parseError) { console.error('Error parsing RSS feeds JSON:', parseError); return false; } } catch (error) { console.error('Error loading RSS feeds:', error); return false; } } // Public API methods getAllFeeds() { return Array.isArray(this.feeds) ? this.feeds : []; } addFeed(feedData) { if (!feedData || !feedData.url) { throw new Error('Feed URL is required'); } // Generate an ID for the feed const id = crypto.randomBytes(8).toString('hex'); const newFeed = { id, name: feedData.name || 'Unnamed Feed', url: feedData.url, autoDownload: !!feedData.autoDownload, filters: Array.isArray(feedData.filters) ? feedData.filters : [], added: new Date().toISOString() }; if (!Array.isArray(this.feeds)) { this.feeds = []; } this.feeds.push(newFeed); // Save the updated feeds this.saveFeeds().catch(err => { console.error('Error saving feeds after adding new feed:', err); }); console.log(`Added new feed: ${newFeed.name} (${newFeed.url})`); return newFeed; } updateFeedConfig(feedId, updates) { if (!feedId || !updates) { return false; } if (!Array.isArray(this.feeds)) { console.error('Feeds is not an array'); return false; } const feedIndex = this.feeds.findIndex(f => f && f.id === feedId); if (feedIndex === -1) { console.error(`Feed with ID ${feedId} not found`); return false; } // Update the feed, preserving the id and added date this.feeds[feedIndex] = { ...this.feeds[feedIndex], ...updates, id: feedId, added: this.feeds[feedIndex].added }; // Save the updated feeds this.saveFeeds().catch(err => { console.error('Error saving feeds after updating feed:', err); }); console.log(`Updated feed: ${this.feeds[feedIndex].name}`); return true; } removeFeed(feedId) { if (!feedId || !Array.isArray(this.feeds)) { return false; } const initialLength = this.feeds.length; this.feeds = this.feeds.filter(f => f && f.id !== feedId); if (this.feeds.length !== initialLength) { // Save the updated feeds this.saveFeeds().catch(err => { console.error('Error saving feeds after removing feed:', err); }); return true; } return false; } getAllItems() { return Array.isArray(this.items) ? this.items : []; } getUndownloadedItems() { if (!Array.isArray(this.items)) { return []; } return this.items.filter(item => item && !item.downloaded && !item.ignored); } filterItems(filters) { if (!filters || !Array.isArray(this.items)) { return []; } return this.items.filter(item => item && this.matchesFilters(item, [filters])); } async downloadItem(item, transmissionClient) { if (!item || !item.torrentLink) { return { success: false, message: 'Invalid item or missing torrent link' }; } if (!transmissionClient) { return { success: false, message: 'Transmission client not available' }; } return new Promise((resolve) => { transmissionClient.addUrl(item.torrentLink, async (err, result) => { if (err) { console.error(`Error adding torrent for ${item.title}:`, err); resolve({ success: false, message: `Error adding torrent: ${err.message}`, result: null }); return; } // Mark the item as downloaded item.downloaded = true; item.downloadDate = new Date().toISOString(); // Save the updated items try { await this.saveItems(); } catch (err) { console.error('Error saving items after download:', err); } console.log(`Successfully added torrent for item: ${item.title}`); resolve({ success: true, message: 'Torrent added successfully', result }); }); }); } } module.exports = RssFeedManager;