From f77002d5f32ed9fff043426aaa5ff94eb3063dca Mon Sep 17 00:00:00 2001 From: John Furrow Date: Sat, 13 Aug 2016 13:12:40 -0700 Subject: [PATCH] Add feed models --- client/scripts/actions/SettingsActions.js | 120 ++++++++++++ client/scripts/constants/ActionTypes.js | 12 ++ client/scripts/constants/EventTypes.js | 10 +- client/scripts/stores/FeedMonitorStore.js | 170 ++++++++++++++++ server/models/Feed.js | 54 +++++ server/models/FeedCollection.js | 227 ++++++++++++++++++++++ server/routes/api.js | 25 +++ 7 files changed, 617 insertions(+), 1 deletion(-) create mode 100644 client/scripts/stores/FeedMonitorStore.js create mode 100644 server/models/Feed.js create mode 100644 server/models/FeedCollection.js diff --git a/client/scripts/actions/SettingsActions.js b/client/scripts/actions/SettingsActions.js index 643f1353..1f2f4382 100644 --- a/client/scripts/actions/SettingsActions.js +++ b/client/scripts/actions/SettingsActions.js @@ -4,6 +4,101 @@ import AppDispatcher from '../dispatcher/AppDispatcher'; import ActionTypes from '../constants/ActionTypes'; const SettingsActions = { + addFeed: (feed) => { + return axios.put('/api/feed-monitor/feeds', feed) + .then((json = {}) => { + return json.data; + }) + .then((data) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_FEED_ADD_SUCCESS, + data + }); + }) + .catch((error) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_FEED_ADD_ERROR, + error + }); + }); + }, + + addRule: (rule) => { + return axios.put('/api/feed-monitor/rules', rule) + .then((json = {}) => { + return json.data; + }) + .then((data) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_RULE_ADD_SUCCESS, + data + }); + }) + .catch((error) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_RULE_ADD_ERROR, + error + }); + }); + }, + + fetchFeedMonitors: (query) => { + return axios.get('/api/feed-monitor', query) + .then((json = {}) => { + return json.data; + }) + .then((data) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITORS_FETCH_SUCCESS, + data + }); + }) + .catch((error) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITORS_FETCH_ERROR, + error + }); + }); + }, + + fetchFeeds: (query) => { + return axios.get('/api/feed-monitor/feeds', query) + .then((json = {}) => { + return json.data; + }) + .then((data) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_FEEDS_FETCH_SUCCESS, + data + }); + }) + .catch((error) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_FEEDS_FETCH_ERROR, + error + }); + }); + }, + + fetchRules: (query) => { + return axios.get('/api/feed-monitor/rules', query) + .then((json = {}) => { + return json.data; + }) + .then((data) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_RULES_FETCH_SUCCESS, + data + }); + }) + .catch((error) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_RULES_FETCH_ERROR, + error + }); + }); + }, + fetchSettings: (property) => { return axios.get('/api/settings', {params: {property}}) .then((json = {}) => { @@ -23,6 +118,31 @@ const SettingsActions = { }); }, + removeFeedMonitor: (id) => { + return axios.delete(`/api/feed-monitor/${id}`) + .then((json = {}) => { + return json.data; + }) + .then((data) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_REMOVE_SUCCESS, + data: { + ...data, + id + } + }); + }) + .catch((error) => { + AppDispatcher.dispatchServerAction({ + type: ActionTypes.SETTINGS_FEED_MONITOR_REMOVE_ERROR, + error: { + ...error, + id + } + }); + }); + }, + saveSettings: (settings, options = {}) => { return axios.patch('/api/settings', settings) .then((json = {}) => { diff --git a/client/scripts/constants/ActionTypes.js b/client/scripts/constants/ActionTypes.js index 674a1c87..bffb8f4c 100644 --- a/client/scripts/constants/ActionTypes.js +++ b/client/scripts/constants/ActionTypes.js @@ -43,6 +43,18 @@ const ActionTypes = { CLIENT_START_TORRENT_SUCCESS: 'CLIENT_START_TORRENT_SUCCESS', CLIENT_STOP_TORRENT_ERROR: 'CLIENT_STOP_TORRENT_ERROR', CLIENT_STOP_TORRENT_SUCCESS: 'CLIENT_STOP_TORRENT_SUCCESS', + SETTINGS_FEED_MONITOR_FEED_ADD_ERROR: 'SETTINGS_FEED_MONITOR_FEED_ADD_ERROR', + SETTINGS_FEED_MONITOR_FEED_ADD_SUCCESS: 'SETTINGS_FEED_MONITOR_FEED_ADD_SUCCESS', + SETTINGS_FEED_MONITOR_FEEDS_FETCH_ERROR: 'SETTINGS_FEED_MONITOR_FEEDS_FETCH_ERROR', + SETTINGS_FEED_MONITOR_FEEDS_FETCH_SUCCESS: 'SETTINGS_FEED_MONITOR_FEEDS_FETCH_SUCCESS', + SETTINGS_FEED_MONITORS_FETCH_ERROR: 'SETTINGS_FEED_MONITORS_FETCH_ERROR', + SETTINGS_FEED_MONITORS_FETCH_SUCCESS: 'SETTINGS_FEED_MONITORS_FETCH_SUCCESS', + SETTINGS_FEED_MONITOR_REMOVE_ERROR: 'SETTINGS_FEED_MONITOR_REMOVE_ERROR', + SETTINGS_FEED_MONITOR_REMOVE_SUCCESS: 'SETTINGS_FEED_MONITOR_REMOVE_SUCCESS', + SETTINGS_FEED_MONITOR_RULE_ADD_ERROR: 'SETTINGS_FEED_MONITOR_RULE_ADD_ERROR', + SETTINGS_FEED_MONITOR_RULE_ADD_SUCCESS: 'SETTINGS_FEED_MONITOR_RULE_ADD_SUCCESS', + SETTINGS_FEED_MONITOR_RULES_FETCH_ERROR: 'SETTINGS_FEED_MONITOR_RULES_FETCH_ERROR', + SETTINGS_FEED_MONITOR_RULES_FETCH_SUCCESS: 'SETTINGS_FEED_MONITOR_RULES_FETCH_SUCCESS', SETTINGS_FETCH_REQUEST_SUCCESS: 'SETTINGS_FETCH_REQUEST_SUCCESS', SETTINGS_FETCH_REQUEST_ERROR: 'SETTINGS_FETCH_REQUEST_ERROR', SETTINGS_SAVE_REQUEST_SUCCESS: 'SETTINGS_SAVE_REQUEST_SUCCESS', diff --git a/client/scripts/constants/EventTypes.js b/client/scripts/constants/EventTypes.js index ccb6693e..03e1082e 100644 --- a/client/scripts/constants/EventTypes.js +++ b/client/scripts/constants/EventTypes.js @@ -32,12 +32,20 @@ const EventTypes = { CLIENT_TORRENT_DETAILS_CHANGE: 'CLIENT_TORRENT_DETAILS_CHANGE', CLIENT_TRANSFER_DATA_REQUEST_SUCCESS: 'CLIENT_TRANSFER_DATA_REQUEST_SUCCESS', CLIENT_TRANSFER_DATA_REQUEST_ERROR: 'CLIENT_TRANSFER_DATA_REQUEST_ERROR', - CLIENT_TRANSFER_HISTORY_REQUEST_SUCCESS: 'CLIENT_TRANSFER_HISTORY_REQUEST_SUCCESS', + CLIENT_TRANSFER_HfSETTINGS_FEED_MONITOR_FETCH_SUCCESSISTORY_REQUEST_SUCCESS: 'CLIENT_TRANSFER_HISTORY_REQUEST_SUCCESS', CLIENT_TRANSFER_HISTORY_REQUEST_ERROR: 'CLIENT_TRANSFER_HISTORY_REQUEST_ERROR', NOTIFICATIONS_CHANGE: 'NOTIFICATIONS_CHANGE', SETTINGS_CHANGE: 'SETTINGS_CHANGE', SETTINGS_SAVE_REQUEST_ERROR: 'SETTINGS_SAVE_REQUEST_ERROR', SETTINGS_SAVE_REQUEST_SUCCESS: 'SETTINGS_SAVE_REQUEST_SUCCESS', + SETTINGS_FEED_MONITOR_FEED_ADD_ERROR: 'SETTINGS_FEED_MONITOR_FEED_ADD_ERROR', + SETTINGS_FEED_MONITOR_FEED_ADD_SUCCESS: 'SETTINGS_FEED_MONITOR_FEED_ADD_SUCCESS', + SETTINGS_FEED_MONITOR_RULE_ADD_ERROR: 'SETTINGS_FEED_MONITOR_RULE_ADD_ERROR', + SETTINGS_FEED_MONITOR_RULE_ADD_SUCCESS: 'SETTINGS_FEED_MONITOR_RULE_ADD_SUCCESS', + SETTINGS_FEED_MONITOR_REMOVE_ERROR: 'SETTINGS_FEED_MONITOR_REMOVE_ERROR', + SETTINGS_FEED_MONITOR_REMOVE_SUCCESS: 'SETTINGS_FEED_MONITOR_REMOVE_SUCCESS', + SETTINGS_FEED_MONITORS_FETCH_ERROR: 'SETTINGS_FEED_MONITORS_FETCH_ERROR', + SETTINGS_FEED_MONITORS_FETCH_SUCCESS: 'SETTINGS_FEED_MONITORS_FETCH_SUCCESS', SETTINGS_FETCH_REQUEST_ERROR: 'SETTINGS_FETCH_REQUEST_ERROR', SETTINGS_FETCH_REQUEST_SUCCESS: 'SETTINGS_FETCH_REQUEST_SUCCESS', UI_CONTEXT_MENU_CHANGE: 'UI_CONTEXT_MENU_CHANGE', diff --git a/client/scripts/stores/FeedMonitorStore.js b/client/scripts/stores/FeedMonitorStore.js new file mode 100644 index 00000000..6297d82c --- /dev/null +++ b/client/scripts/stores/FeedMonitorStore.js @@ -0,0 +1,170 @@ +import ActionTypes from '../constants/ActionTypes'; +import AppDispatcher from '../dispatcher/AppDispatcher'; +import SettingsActions from '../actions/SettingsActions'; +import BaseStore from './BaseStore'; +import EventTypes from '../constants/EventTypes'; + +class FeedsStoreClass extends BaseStore { + constructor() { + super(); + this.feeds = []; + this.rules = []; + } + + addFeed(feed) { + SettingsActions.addFeed(feed); + } + + addRule(feed) { + SettingsActions.addRule(feed); + } + + fetchFeedMonitors(query) { + SettingsActions.fetchFeedMonitors(query); + } + + fetchFeeds(query) { + SettingsActions.fetchFeeds(query); + } + + fetchRules(query) { + SettingsActions.fetchRules(query); + } + + getFeeds() { + return this.feeds; + } + + getRules() { + return this.rules; + } + + handleFeedAddError(error) { + this.emit(EventTypes.SETTINGS_FEED_MONITOR_FEED_ADD_ERROR, error); + } + + handleFeedAddSuccess() { + this.fetchFeedMonitors(); + this.emit(EventTypes.SETTINGS_FEED_MONITOR_FEED_ADD_SUCCESS); + } + + handleRuleAddError(error) { + this.emit(EventTypes.SETTINGS_FEED_MONITOR_RULE_ADD_ERROR, error); + } + + handleRuleAddSuccess() { + this.fetchFeedMonitors(); + this.emit(EventTypes.SETTINGS_FEED_MONITOR_RULE_ADD_SUCCESS); + } + + handleFeedMonitorsFetchError(error) { + this.emit(EventTypes.SETTINGS_FEED_MONITORS_FETCH_ERROR, error); + } + + handleFeedMonitorsFetchSuccess(feedMonitors) { + this.setFeeds(feedMonitors.feeds); + this.setRules(feedMonitors.rules); + this.emit(EventTypes.SETTINGS_FEED_MONITORS_FETCH_SUCCESS); + } + + handleFeedMonitorRemoveError(id) { + this.emit(EventTypes.SETTINGS_FEED_MONITOR_REMOVE_ERROR, id); + } + + handleFeedMonitorRemoveSuccess(id) { + this.fetchFeedMonitors(); + this.emit(EventTypes.SETTINGS_FEED_MONITOR_REMOVE_SUCCESS, id); + } + + handleFeedsFetchError(error) { + this.emit(EventTypes.SETTINGS_FEED_MONITOR_FEEDS_FETCH_ERROR, error); + } + + handleFeedsFetchSuccess(feeds) { + this.setFeeds(feeds); + this.emit(EventTypes.SETTINGS_FEED_MONITOR_FEEDS_FETCH_SUCCESS); + } + + handleRulesFetchError(error) { + this.emit(EventTypes.SETTINGS_FEED_MONITOR_RULES_FETCH_ERROR, error); + } + + handleRulesFetchSuccess(rules) { + this.setRules(rules); + this.emit(EventTypes.SETTINGS_FEED_MONITOR_RULES_FETCH_SUCCESS); + } + + removeFeed(id) { + SettingsActions.removeFeedMonitor(id); + } + + removeRule(id) { + SettingsActions.removeFeedMonitor(id); + } + + setItems(type, items) { + if (items == null) { + this[type] = []; + return; + } + + this[type] = items.sort((a, b) => { + return a.label.localeCompare(b.label); + }); + } + + setFeeds(feeds) { + this.setItems('feeds', feeds); + } + + setRules(rules) { + this.setItems('rules', rules); + } +} + +let FeedsStore = new FeedsStoreClass(); + +FeedsStore.dispatcherID = AppDispatcher.register((payload) => { + const {action, source} = payload; + + switch (action.type) { + case ActionTypes.SETTINGS_FEED_MONITOR_FEED_ADD_ERROR: + FeedsStore.handleFeedAddError(action.error); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_FEED_ADD_SUCCESS: + FeedsStore.handleFeedAddSuccess(); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_RULE_ADD_ERROR: + FeedsStore.handleRuleAddError(action.error); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_RULE_ADD_SUCCESS: + FeedsStore.handleRuleAddSuccess(); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_REMOVE_ERROR: + FeedsStore.handleFeedMonitorRemoveError(action.error.id); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_REMOVE_SUCCESS: + FeedsStore.handleFeedMonitorRemoveSuccess(action.data.id); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_FEEDS_FETCH_ERROR: + FeedsStore.handleFeedsFetchError(action.error); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_FEEDS_FETCH_SUCCESS: + FeedsStore.handleFeedsFetchSuccess(action.data); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_RULES_FETCH_ERROR: + FeedsStore.handleRulesFetchError(action.error); + break; + case ActionTypes.SETTINGS_FEED_MONITOR_RULES_FETCH_SUCCESS: + FeedsStore.handleRulesFetchSuccess(action.data); + break; + case ActionTypes.SETTINGS_FEED_MONITORS_FETCH_ERROR: + FeedsStore.handleFeedMonitorsFetchError(action.error); + break; + case ActionTypes.SETTINGS_FEED_MONITORS_FETCH_SUCCESS: + FeedsStore.handleFeedMonitorsFetchSuccess(action.data); + break; + } +}); + +export default FeedsStore; diff --git a/server/models/Feed.js b/server/models/Feed.js new file mode 100644 index 00000000..bd6fe8d2 --- /dev/null +++ b/server/models/Feed.js @@ -0,0 +1,54 @@ +'use strict'; + +let FeedSub = require('feedsub'); + +class Feed { + constructor(options) { + this.options = options || {}; + + if (!options.url) { + console.error('Feed URL must be defined.'); + return null; + } + + this.items = []; + this.maxItemHistory = options.maxItemHistory || 10; + this.reader = new FeedSub(options.url, { + autoStart: true, + emitOnStart: true, + interval: options.interval || 15 + }); + + this.initReader(); + } + + getRecentItems() { + return this.items; + } + + handleFeedItem(item) { + let arrayAction = 'push'; + + if (this.items.length >= this.maxItemHistory) { + arrayAction = 'shift'; + } + + this.items[arrayAction](item); + + this.options.onNewItem({feed: this.options, torrent: item}); + } + + initReader() { + this.reader.on('item', this.handleFeedItem.bind(this)); + this.reader.on('error', error => { + console.log('Feed reader error:', error); + }); + this.reader.start(); + } + + stop() { + this.reader.stop(); + } +} + +module.exports = Feed; diff --git a/server/models/FeedCollection.js b/server/models/FeedCollection.js new file mode 100644 index 00000000..a6d4cd73 --- /dev/null +++ b/server/models/FeedCollection.js @@ -0,0 +1,227 @@ +'use strict'; + +let _ = require('lodash'); +let Datastore = require('nedb'); + +let client = require('./client'); +let config = require('../../config'); +let Feed = require('./Feed'); + +class FeedCollection { + constructor(opts) { + this.opts = opts || {}; + + let defaultFeedOptions = { + maxItemHistory: 50, + onNewItem: this.handleNewItem + }; + + this.feeds = []; + this.interval = null; + this.isDBReady = false; + this.rules = {}; + this.db = this.loadDatabase(); + + this.init(); + } + + addFeed(feed, callback) { + this.addItem('feed', feed, (newDoc) => { + newDoc.onNewItem = this.handleNewItem.bind(this); + this.feeds.push(new Feed(newDoc)); + callback(newDoc); + }); + } + + addItem(type, item, callback) { + if (!this.isDBReady) { + return; + } + + this.db.insert(Object.assign(item, {type}), (err, newDoc) => { + if (err) { + callback(null, err); + return; + } + + callback(newDoc); + }); + } + + addRule(rule, callback) { + this.addItem('rule', rule, callback); + } + + downloadTorrent(matchedTorrent, downloadRule) { + this.db.find({type: 'matchedTorrents'}, (err, previouslyMatchedTorrents) => { + if (err) { + return; + } + + let shouldDownload = !this.isTorrentURLAlreadyDownloaded( + matchedTorrent.torrent.link, previouslyMatchedTorrents[0] || {}); + + if (shouldDownload) { + client.addUrls({ + urls: matchedTorrent.torrent.link, + destination: downloadRule.destination, + start: downloadRule.startOnLoad, + tags: downloadRule.tags + }, () => { + this.db.update({type: 'matchedTorrents'}, + {$push: {urls: matchedTorrent.torrent.link}}, {upsert: true}); + + this.db.update({_id: downloadRule._id}, {$inc: {count: 1}}, + {upsert: true}); + + this.db.update({_id: matchedTorrent.feed._id}, {$inc: {count: 1}}, + {upsert: true}); + }); + } + }); + } + + getAll(query, callback) { + query = query || {}; + + this.db.find({}, (err, docs) => { + if (err) { + callback(null, err); + return; + } + + callback(docs.reduce((memo, item) => { + let type = `${item.type}s`; + + if (memo[type] == null) { + memo[type] = []; + } + + memo[type].push(item); + + return memo; + }, {})); + }); + } + + getFeeds(query, callback) { + this.getItem('feed', query, callback); + } + + getItem(type, query, callback) { + query = query || {}; + + this.db.find(Object.assign(query, {type}), (err, docs) => { + if (err) { + callback(null, err); + return; + } + + callback(docs); + }); + } + + getRules(query, callback) { + this.getItem('rule', query, callback); + } + + handleNewItem(feedItem) { + let downloadRules = this.rules[feedItem.feed._id]; + + if (downloadRules) { + downloadRules.forEach((downloadRule) => { + if (!downloadRule.field || !downloadRule.match) { + return; + } + + let isFeedMatched = feedItem.torrent[downloadRule.field] + .match(new RegExp(downloadRule.match, 'gi')); + + if (!isFeedMatched) { + return; + } + + let isFeedExcluded = downloadRule.exclude + && feedItem.torrent[downloadRule.field] + .match(new RegExp(downloadRule.exclude, 'gi')); + + if (!isFeedExcluded) { + this.downloadTorrent(feedItem, downloadRule); + } + }); + } + } + + init() { + this.db.find({}, (err, docs) => { + if (err) { + return ; + } + + let newItemHandler = this.handleNewItem.bind(this); + + docs.forEach((doc) => { + if (doc.type === 'feed') { + doc.onNewItem = newItemHandler; + this.feeds.push(new Feed(doc)); + } else if (doc.type === 'rule') { + if (this.rules[doc.feedID] == null) { + this.rules[doc.feedID] = []; + } + + this.rules[doc.feedID].push(doc); + } + }); + }); + } + + loadDatabase() { + let db = new Datastore({ + autoload: true, + filename: `${config.dbPath}settings/feeds.db` + }); + + this.isDBReady = true; + return db; + } + + removeItem(id, callback) { + let indexToRemove = -1; + let itemToRemove = this.feeds.find((feed, index) => { + indexToRemove = index; + return feed.options._id === id; + }); + + if (itemToRemove != null) { + itemToRemove.stop(); + this.feeds.splice(indexToRemove, 1); + } + + this.db.remove({_id: id}, {}, (err, docs) => { + if (err) { + callback(null, err); + return; + } + + callback(docs); + }); + } + + isTorrentURLAlreadyDownloaded(torrentURL, downloadedTorrents) { + let isAlreadyDownloaded = false; + + if (!!downloadedTorrents.urls && downloadedTorrents.urls.length > 0) { + downloadedTorrents.urls.some(url => { + if (torrentURL === url) { + isAlreadyDownloaded = true; + } + + return isAlreadyDownloaded; + }); + } + + return isAlreadyDownloaded; + } +} + +module.exports = new FeedCollection(); diff --git a/server/routes/api.js b/server/routes/api.js index 0d1a5890..1d9d3a56 100644 --- a/server/routes/api.js +++ b/server/routes/api.js @@ -5,6 +5,7 @@ let express = require('express'); let ajaxUtil = require('../util/ajaxUtil'); let client = require('../models/client'); let clientRoutes = require('./client'); +let FeedCollection = require('../models/FeedCollection'); let history = require('../models/history'); let passport = require('passport'); let router = express.Router(); @@ -16,6 +17,30 @@ router.use('/', passport.authenticate('jwt', {session: false})); router.use('/client', clientRoutes); +router.delete('/feed-monitor/:id', (req, res, next) => { + FeedCollection.removeItem(req.params.id, ajaxUtil.getResponseFn(res)); +}); + +router.get('/feed-monitor', (req, res, next) => { + FeedCollection.getAll(req.body.query, ajaxUtil.getResponseFn(res)); +}); + +router.get('/feed-monitor/feeds', (req, res, next) => { + FeedCollection.getFeeds(req.body.query, ajaxUtil.getResponseFn(res)); +}); + +router.put('/feed-monitor/feeds', (req, res, next) => { + FeedCollection.addFeed(req.body, ajaxUtil.getResponseFn(res)); +}); + +router.get('/feed-monitor/rules', (req, res, next) => { + FeedCollection.getRules(req.body.query, ajaxUtil.getResponseFn(res)); +}); + +router.put('/feed-monitor/rules', (req, res, next) => { + FeedCollection.addRule(req.body, ajaxUtil.getResponseFn(res)); +}); + router.get('/history', (req, res, next) => { history.get(req.query, ajaxUtil.getResponseFn(res)); });