456 lines
12 KiB
JavaScript
456 lines
12 KiB
JavaScript
// rssFeedManager.js
|
|
const fs = require('fs').promises;
|
|
const path = require('path');
|
|
const fetch = require('node-fetch');
|
|
const xml2js = require('xml2js');
|
|
const crypto = require('crypto');
|
|
|
|
class RssFeedManager {
|
|
constructor(config) {
|
|
this.config = config;
|
|
this.feeds = config.feeds || [];
|
|
this.items = [];
|
|
this.updateIntervalId = null;
|
|
this.updateIntervalMinutes = config.updateIntervalMinutes || 60;
|
|
this.parser = new xml2js.Parser({ explicitArray: false });
|
|
this.dataPath = path.join(__dirname, '..', 'data');
|
|
}
|
|
|
|
async start() {
|
|
if (this.updateIntervalId) {
|
|
return;
|
|
}
|
|
|
|
// Run update immediately
|
|
await this.updateAllFeeds();
|
|
|
|
// Then set up interval
|
|
this.updateIntervalId = setInterval(async () => {
|
|
await this.updateAllFeeds();
|
|
}, this.updateIntervalMinutes * 60 * 1000);
|
|
|
|
console.log(`RSS feed manager started, interval: ${this.updateIntervalMinutes} minutes`);
|
|
}
|
|
|
|
stop() {
|
|
if (this.updateIntervalId) {
|
|
clearInterval(this.updateIntervalId);
|
|
this.updateIntervalId = null;
|
|
console.log('RSS feed manager stopped');
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async updateAllFeeds() {
|
|
console.log('Updating all RSS feeds...');
|
|
|
|
const results = [];
|
|
|
|
for (const feed of this.feeds) {
|
|
try {
|
|
const result = await this.updateFeed(feed);
|
|
results.push({
|
|
feedId: feed.id,
|
|
success: true,
|
|
newItems: result.newItems
|
|
});
|
|
} catch (error) {
|
|
console.error(`Error updating feed ${feed.id} (${feed.url}):`, error.message);
|
|
results.push({
|
|
feedId: feed.id,
|
|
success: false,
|
|
error: error.message
|
|
});
|
|
}
|
|
}
|
|
|
|
// Save updated items
|
|
await this.saveItems();
|
|
|
|
console.log('RSS feed update completed');
|
|
return results;
|
|
}
|
|
|
|
async updateFeed(feed) {
|
|
console.log(`Updating feed: ${feed.name} (${feed.url})`);
|
|
|
|
try {
|
|
const response = await fetch(feed.url);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
const xml = await response.text();
|
|
const result = await this.parseXml(xml);
|
|
|
|
const rssItems = this.extractItems(result, feed);
|
|
const newItems = this.processNewItems(rssItems, feed);
|
|
|
|
console.log(`Found ${rssItems.length} items, ${newItems.length} new items in feed: ${feed.name}`);
|
|
|
|
return {
|
|
totalItems: rssItems.length,
|
|
newItems: newItems.length
|
|
};
|
|
} catch (error) {
|
|
console.error(`Error updating feed ${feed.url}:`, error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
parseXml(xml) {
|
|
return new Promise((resolve, reject) => {
|
|
this.parser.parseString(xml, (error, result) => {
|
|
if (error) {
|
|
reject(error);
|
|
} else {
|
|
resolve(result);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
extractItems(parsedXml, feed) {
|
|
try {
|
|
// Handle standard RSS 2.0
|
|
if (parsedXml.rss && parsedXml.rss.channel) {
|
|
const channel = parsedXml.rss.channel;
|
|
const items = Array.isArray(channel.item) ? channel.item : [channel.item].filter(Boolean);
|
|
return items.map(item => this.normalizeRssItem(item, feed));
|
|
}
|
|
|
|
// Handle Atom
|
|
if (parsedXml.feed && parsedXml.feed.entry) {
|
|
const entries = Array.isArray(parsedXml.feed.entry) ? parsedXml.feed.entry : [parsedXml.feed.entry].filter(Boolean);
|
|
return entries.map(entry => this.normalizeAtomItem(entry, feed));
|
|
}
|
|
|
|
return [];
|
|
} catch (error) {
|
|
console.error('Error extracting items from XML:', error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
normalizeRssItem(item, feed) {
|
|
// Create a unique ID for the item
|
|
const idContent = `${feed.id}:${item.title}:${item.pubDate || ''}:${item.link || ''}`;
|
|
const id = crypto.createHash('md5').update(idContent).digest('hex');
|
|
|
|
// Extract enclosure (torrent link)
|
|
let torrentLink = item.link || '';
|
|
let fileSize = 0;
|
|
|
|
if (item.enclosure) {
|
|
torrentLink = item.enclosure.$ ? item.enclosure.$.url : item.enclosure.url || torrentLink;
|
|
fileSize = item.enclosure.$ ? parseInt(item.enclosure.$.length || 0, 10) : parseInt(item.enclosure.length || 0, 10);
|
|
}
|
|
|
|
// Handle custom namespaces (common in torrent feeds)
|
|
let category = '';
|
|
let size = fileSize;
|
|
|
|
if (item.category) {
|
|
category = Array.isArray(item.category) ? item.category[0] : item.category;
|
|
}
|
|
|
|
// Some feeds use torrent:contentLength
|
|
if (item['torrent:contentLength']) {
|
|
size = parseInt(item['torrent:contentLength'], 10);
|
|
}
|
|
|
|
return {
|
|
id,
|
|
feedId: feed.id,
|
|
title: item.title || 'Untitled',
|
|
link: item.link || '',
|
|
torrentLink: torrentLink,
|
|
pubDate: item.pubDate || new Date().toISOString(),
|
|
category: category,
|
|
description: item.description || '',
|
|
size: size || 0,
|
|
downloaded: false,
|
|
ignored: false,
|
|
added: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
normalizeAtomItem(entry, feed) {
|
|
// Create a unique ID for the item
|
|
const idContent = `${feed.id}:${entry.title}:${entry.updated || ''}:${entry.id || ''}`;
|
|
const id = crypto.createHash('md5').update(idContent).digest('hex');
|
|
|
|
// Extract link
|
|
let link = '';
|
|
let torrentLink = '';
|
|
|
|
if (entry.link) {
|
|
if (Array.isArray(entry.link)) {
|
|
const links = entry.link;
|
|
link = links.find(l => l.$.rel === 'alternate')?.$.href || links[0]?.$.href || '';
|
|
torrentLink = links.find(l => l.$.type && l.$.type.includes('torrent'))?.$.href || link;
|
|
} else {
|
|
link = entry.link.$.href || '';
|
|
torrentLink = link;
|
|
}
|
|
}
|
|
|
|
return {
|
|
id,
|
|
feedId: feed.id,
|
|
title: entry.title || 'Untitled',
|
|
link: link,
|
|
torrentLink: torrentLink,
|
|
pubDate: entry.updated || entry.published || new Date().toISOString(),
|
|
category: entry.category?.$.term || '',
|
|
description: entry.summary || entry.content || '',
|
|
size: 0, // Atom doesn't typically include file size
|
|
downloaded: false,
|
|
ignored: false,
|
|
added: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
processNewItems(rssItems, feed) {
|
|
const newItems = [];
|
|
|
|
for (const item of rssItems) {
|
|
// Check if item already exists in our list
|
|
const existingItem = this.items.find(i => i.id === item.id);
|
|
|
|
if (!existingItem) {
|
|
// Add new item to our list
|
|
this.items.push(item);
|
|
newItems.push(item);
|
|
|
|
// Auto-download if enabled and matches filters
|
|
if (feed.autoDownload && this.matchesFilters(item, feed.filters)) {
|
|
this.queueItemForDownload(item);
|
|
}
|
|
}
|
|
}
|
|
|
|
return newItems;
|
|
}
|
|
|
|
matchesFilters(item, filters) {
|
|
if (!filters || filters.length === 0) {
|
|
return true;
|
|
}
|
|
|
|
// Check if the item matches any of the filters
|
|
return filters.some(filter => {
|
|
// Title check
|
|
if (filter.title && !item.title.toLowerCase().includes(filter.title.toLowerCase())) {
|
|
return false;
|
|
}
|
|
|
|
// Category check
|
|
if (filter.category && !item.category.toLowerCase().includes(filter.category.toLowerCase())) {
|
|
return false;
|
|
}
|
|
|
|
// Size check
|
|
if (filter.minSize && item.size < filter.minSize) {
|
|
return false;
|
|
}
|
|
|
|
if (filter.maxSize && item.size > filter.maxSize) {
|
|
return false;
|
|
}
|
|
|
|
// All checks passed
|
|
return true;
|
|
});
|
|
}
|
|
|
|
queueItemForDownload(item) {
|
|
// Mark the item as queued for download
|
|
console.log(`Auto-downloading item: ${item.title}`);
|
|
|
|
// This would be implemented to add to Transmission
|
|
// But we need a reference to the Transmission client
|
|
// In a real implementation, this might publish to a queue that's consumed elsewhere
|
|
item.downloadQueued = true;
|
|
}
|
|
|
|
async saveItems() {
|
|
try {
|
|
// Create data directory if it doesn't exist
|
|
await fs.mkdir(this.dataPath, { recursive: true });
|
|
|
|
// Save items to file
|
|
await fs.writeFile(
|
|
path.join(this.dataPath, 'rss-items.json'),
|
|
JSON.stringify(this.items, null, 2),
|
|
'utf8'
|
|
);
|
|
|
|
console.log(`Saved ${this.items.length} RSS items to disk`);
|
|
return true;
|
|
} catch (error) {
|
|
console.error('Error saving RSS items:', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async saveConfig() {
|
|
try {
|
|
// Create data directory if it doesn't exist
|
|
await fs.mkdir(this.dataPath, { recursive: true });
|
|
|
|
// Save feeds to file
|
|
await fs.writeFile(
|
|
path.join(this.dataPath, 'rss-feeds.json'),
|
|
JSON.stringify(this.feeds, null, 2),
|
|
'utf8'
|
|
);
|
|
|
|
console.log(`Saved ${this.feeds.length} RSS feeds to disk`);
|
|
return true;
|
|
} catch (error) {
|
|
console.error('Error saving RSS feeds:', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async loadItems() {
|
|
try {
|
|
const filePath = path.join(this.dataPath, 'rss-items.json');
|
|
|
|
// Check if file exists
|
|
try {
|
|
await fs.access(filePath);
|
|
} catch (error) {
|
|
console.log('No saved RSS items found');
|
|
return false;
|
|
}
|
|
|
|
// Load items from file
|
|
const data = await fs.readFile(filePath, 'utf8');
|
|
this.items = JSON.parse(data);
|
|
|
|
console.log(`Loaded ${this.items.length} RSS items from disk`);
|
|
return true;
|
|
} catch (error) {
|
|
console.error('Error loading RSS items:', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Public API methods
|
|
|
|
getAllFeeds() {
|
|
return this.feeds;
|
|
}
|
|
|
|
addFeed(feedData) {
|
|
// Generate an ID for the feed
|
|
const id = crypto.randomBytes(8).toString('hex');
|
|
|
|
const newFeed = {
|
|
id,
|
|
name: feedData.name,
|
|
url: feedData.url,
|
|
autoDownload: feedData.autoDownload || false,
|
|
filters: feedData.filters || [],
|
|
added: new Date().toISOString()
|
|
};
|
|
|
|
this.feeds.push(newFeed);
|
|
|
|
console.log(`Added new feed: ${newFeed.name} (${newFeed.url})`);
|
|
|
|
return newFeed;
|
|
}
|
|
|
|
updateFeedConfig(feedId, updates) {
|
|
const feedIndex = this.feeds.findIndex(f => f.id === feedId);
|
|
|
|
if (feedIndex === -1) {
|
|
return false;
|
|
}
|
|
|
|
// Update the feed, preserving the id and added date
|
|
this.feeds[feedIndex] = {
|
|
...this.feeds[feedIndex],
|
|
...updates,
|
|
id: feedId,
|
|
added: this.feeds[feedIndex].added
|
|
};
|
|
|
|
console.log(`Updated feed: ${this.feeds[feedIndex].name}`);
|
|
|
|
return true;
|
|
}
|
|
|
|
removeFeed(feedId) {
|
|
const initialLength = this.feeds.length;
|
|
this.feeds = this.feeds.filter(f => f.id !== feedId);
|
|
|
|
return this.feeds.length !== initialLength;
|
|
}
|
|
|
|
getAllItems() {
|
|
return this.items;
|
|
}
|
|
|
|
getUndownloadedItems() {
|
|
return this.items.filter(item => !item.downloaded && !item.ignored);
|
|
}
|
|
|
|
filterItems(filters) {
|
|
return this.items.filter(item => this.matchesFilters(item, [filters]));
|
|
}
|
|
|
|
async downloadItem(item, transmissionClient) {
|
|
if (!item || !item.torrentLink) {
|
|
return {
|
|
success: false,
|
|
message: 'Invalid item or missing torrent link'
|
|
};
|
|
}
|
|
|
|
if (!transmissionClient) {
|
|
return {
|
|
success: false,
|
|
message: 'Transmission client not available'
|
|
};
|
|
}
|
|
|
|
return new Promise((resolve) => {
|
|
transmissionClient.addUrl(item.torrentLink, (err, result) => {
|
|
if (err) {
|
|
console.error(`Error adding torrent for ${item.title}:`, err);
|
|
resolve({
|
|
success: false,
|
|
message: `Error adding torrent: ${err.message}`,
|
|
result: null
|
|
});
|
|
return;
|
|
}
|
|
|
|
// Mark the item as downloaded
|
|
item.downloaded = true;
|
|
item.downloadDate = new Date().toISOString();
|
|
|
|
// Save the updated items
|
|
this.saveItems().catch(err => {
|
|
console.error('Error saving items after download:', err);
|
|
});
|
|
|
|
console.log(`Successfully added torrent for item: ${item.title}`);
|
|
|
|
resolve({
|
|
success: true,
|
|
message: 'Torrent added successfully',
|
|
result
|
|
});
|
|
});
|
|
});
|
|
}
|
|
}
|
|
|
|
module.exports = RssFeedManager; |