From b0ff983600e9176ee6bc72e4990f5a7f697ffbc2 Mon Sep 17 00:00:00 2001 From: Jesse Chan Date: Wed, 21 Oct 2020 19:50:47 +0800 Subject: [PATCH] server: migrate feedService to TypeScript --- .../src/javascript/actions/SettingsActions.ts | 58 +-- .../modals/feeds-modal/DownloadRulesTab.tsx | 34 +- .../modals/feeds-modal/FeedsModal.tsx | 4 +- .../modals/feeds-modal/FeedsTab.tsx | 62 +-- .../sidebar/NotificationsButton.tsx | 47 +-- .../src/javascript/constants/ServerActions.ts | 10 +- .../src/javascript/i18n/strings.compiled.json | 18 + client/src/javascript/i18n/strings.json | 3 + client/src/javascript/stores/FeedsStore.ts | 133 ++---- package-lock.json | 19 + package.json | 1 + server/models/Feed.js | 75 ---- server/models/FeedReader.ts | 74 ++++ server/routes/api/feed-monitor.ts | 173 +++++++- server/services/feedService.js | 365 ----------------- server/services/feedService.ts | 386 ++++++++++++++++++ server/services/notificationService.ts | 2 +- server/util/feedUtil.ts | 88 ++++ shared/types/Feed.ts | 34 ++ shared/types/Notification.ts | 21 +- shared/types/api/feed-monitor.ts | 7 + 21 files changed, 929 insertions(+), 685 deletions(-) delete mode 100644 server/models/Feed.js create mode 100644 server/models/FeedReader.ts delete mode 100644 server/services/feedService.js create mode 100644 server/services/feedService.ts create mode 100644 server/util/feedUtil.ts create mode 100644 shared/types/Feed.ts create mode 100644 shared/types/api/feed-monitor.ts diff --git a/client/src/javascript/actions/SettingsActions.ts b/client/src/javascript/actions/SettingsActions.ts index 402c1d55..729ee1af 100644 --- a/client/src/javascript/actions/SettingsActions.ts +++ b/client/src/javascript/actions/SettingsActions.ts @@ -1,19 +1,19 @@ import axios from 'axios'; +import type {AddFeedOptions, AddRuleOptions, ModifyFeedOptions} from '@shared/types/api/feed-monitor'; import type {SetFloodSettingsOptions} from '@shared/types/api/index'; import AppDispatcher from '../dispatcher/AppDispatcher'; import ConfigStore from '../stores/ConfigStore'; -import type {Feed, Rule} from '../stores/FeedsStore'; import type {SettingsSaveRequestSuccessAction} from '../constants/ServerActions'; const baseURI = ConfigStore.getBaseURI(); const SettingsActions = { - addFeed: (feed: Feed) => + addFeed: (options: AddFeedOptions) => axios - .put(`${baseURI}api/feed-monitor/feeds`, feed) + .put(`${baseURI}api/feed-monitor/feeds`, options) .then((json) => json.data) .then( () => { @@ -29,9 +29,9 @@ const SettingsActions = { }, ), - modifyFeed: (id: Feed['_id'], feed: Feed) => + modifyFeed: (id: string, options: ModifyFeedOptions) => axios - .put(`${baseURI}api/feed-monitor/feeds/${id}`, feed) + .put(`${baseURI}api/feed-monitor/feeds/${id}`, options) .then((json) => json.data) .then( () => { @@ -47,9 +47,9 @@ const SettingsActions = { }, ), - addRule: (rule: Rule) => + addRule: (options: AddRuleOptions) => axios - .put(`${baseURI}api/feed-monitor/rules`, rule) + .put(`${baseURI}api/feed-monitor/rules`, options) .then((json) => json.data) .then( () => { @@ -84,28 +84,13 @@ const SettingsActions = { }, ), - fetchFeeds: (query: string) => + fetchItems: ({id, search}: {id: string; search: string}) => axios - .get(`${baseURI}api/feed-monitor/feeds`, {params: query}) - .then((json) => json.data) - .then( - (data) => { - AppDispatcher.dispatchServerAction({ - type: 'SETTINGS_FEED_MONITOR_FEEDS_FETCH_SUCCESS', - data, - }); + .get(`${baseURI}api/feed-monitor/feeds/${id}/items`, { + params: { + search, }, - (error) => { - AppDispatcher.dispatchServerAction({ - type: 'SETTINGS_FEED_MONITOR_FEEDS_FETCH_ERROR', - error, - }); - }, - ), - - fetchItems: (query: {params: {id: string; search: string}}) => - axios - .get(`${baseURI}api/feed-monitor/items`, query) + }) .then((json) => json.data) .then( (data) => { @@ -122,25 +107,6 @@ const SettingsActions = { }, ), - fetchRules: (query: string) => - axios - .get(`${baseURI}api/feed-monitor/rules`, {params: query}) - .then((json) => json.data) - .then( - (data) => { - AppDispatcher.dispatchServerAction({ - type: 'SETTINGS_FEED_MONITOR_RULES_FETCH_SUCCESS', - data, - }); - }, - (error) => { - AppDispatcher.dispatchServerAction({ - type: 'SETTINGS_FEED_MONITOR_RULES_FETCH_ERROR', - error, - }); - }, - ), - fetchSettings: (property?: Record) => axios .get(`${baseURI}api/settings`, {params: {property}}) diff --git a/client/src/javascript/components/modals/feeds-modal/DownloadRulesTab.tsx b/client/src/javascript/components/modals/feeds-modal/DownloadRulesTab.tsx index 13dca949..683abc2a 100644 --- a/client/src/javascript/components/modals/feeds-modal/DownloadRulesTab.tsx +++ b/client/src/javascript/components/modals/feeds-modal/DownloadRulesTab.tsx @@ -2,6 +2,9 @@ import {defineMessages, FormattedMessage, injectIntl, WrappedComponentProps} fro import React from 'react'; import throttle from 'lodash/throttle'; +import type {AddRuleOptions} from '@shared/types/api/feed-monitor'; +import type {Feed, Rule} from '@shared/types/Feed'; + import { Button, Checkbox, @@ -16,17 +19,16 @@ import { Textbox, } from '../../../ui'; import connectStores from '../../../util/connectStores'; -import Edit from '../../icons/Edit'; import Checkmark from '../../icons/Checkmark'; import Close from '../../icons/Close'; -import FeedsStore, {FeedsStoreClass} from '../../../stores/FeedsStore'; +import Edit from '../../icons/Edit'; +import FeedsStore from '../../../stores/FeedsStore'; import FilesystemBrowserTextbox from '../../general/filesystem/FilesystemBrowserTextbox'; import ModalFormSectionHeader from '../ModalFormSectionHeader'; +import SettingsActions from '../../../actions/SettingsActions'; import TagSelect from '../../general/form-elements/TagSelect'; import * as validators from '../../../util/validators'; -import type {Feeds, Rule, Rules} from '../../../stores/FeedsStore'; - type ValidatedFields = 'destination' | 'feedID' | 'label' | 'match' | 'exclude'; interface RuleFormData extends Omit { @@ -35,15 +37,15 @@ interface RuleFormData extends Omit { } interface DownloadRulesTabProps extends WrappedComponentProps { - feeds: Feeds; - rules: Rules; + feeds: Array; + rules: Array; } interface DownloadRulesTabStates { errors?: { [field in ValidatedFields]?: string; }; - currentlyEditingRule: Rule | null; + currentlyEditingRule: Partial | null; doesPatternMatchTest: boolean; } @@ -141,7 +143,7 @@ class DownloadRulesTab extends React.Component) { const {doesPatternMatchTest, currentlyEditingRule} = this.state; return ( @@ -301,6 +302,7 @@ class DownloadRulesTab extends React.Component + {': '} {rule.exclude} ); @@ -316,6 +318,7 @@ class DownloadRulesTab extends React.Component + {': '} {tagNodes} ); @@ -347,10 +350,13 @@ class DownloadRulesTab extends React.Component
  • + interactive-list__detail interactive-list__detail--tertiary" + style={{maxWidth: '50%', overflow: 'hidden', textOverflow: 'ellipsis'}}> + {': '} {rule.match}
    • +
    {excludeNode} {tags} @@ -415,9 +421,9 @@ class DownloadRulesTab extends React.Component { componentDidMount() { - FeedsStoreClass.fetchFeedMonitors(); + SettingsActions.fetchFeedMonitors(); } render() { diff --git a/client/src/javascript/components/modals/feeds-modal/FeedsTab.tsx b/client/src/javascript/components/modals/feeds-modal/FeedsTab.tsx index 2f8175b6..2f7d047a 100644 --- a/client/src/javascript/components/modals/feeds-modal/FeedsTab.tsx +++ b/client/src/javascript/components/modals/feeds-modal/FeedsTab.tsx @@ -2,6 +2,8 @@ import {defineMessages, FormattedMessage, injectIntl, WrappedComponentProps} fro import React from 'react'; import throttle from 'lodash/throttle'; +import type {Feed, Item} from '@shared/types/Feed'; + import { Button, Checkbox, @@ -14,16 +16,15 @@ import { SelectItem, Textbox, } from '../../../ui'; -import Edit from '../../icons/Edit'; +import connectStores from '../../../util/connectStores'; import Close from '../../icons/Close'; -import FeedsStore, {FeedsStoreClass} from '../../../stores/FeedsStore'; +import Edit from '../../icons/Edit'; +import FeedsStore from '../../../stores/FeedsStore'; import {minToHumanReadable} from '../../../i18n/languages'; import ModalFormSectionHeader from '../ModalFormSectionHeader'; -import * as validators from '../../../util/validators'; +import SettingsActions from '../../../actions/SettingsActions'; import UIActions from '../../../actions/UIActions'; -import connectStores from '../../../util/connectStores'; - -import type {Feed, Feeds, Items} from '../../../stores/FeedsStore'; +import * as validators from '../../../util/validators'; interface IntervalMultiplier { displayName: string; @@ -33,12 +34,15 @@ interface IntervalMultiplier { type ValidatedFields = 'url' | 'label' | 'interval'; interface FeedFormData extends Feed { + url: string; + label: string; + interval: number; intervalMultiplier: number; } interface FeedsTabProps extends WrappedComponentProps { - feeds: Feeds; - items: Items; + feeds: Array; + items: Array; } interface FeedsTabStates { @@ -46,7 +50,7 @@ interface FeedsTabStates { [field in ValidatedFields]?: string; }; intervalMultipliers: Array; - currentlyEditingFeed: Feed | null; + currentlyEditingFeed: Partial | null; selectedFeedID: string | null; } @@ -148,20 +152,24 @@ class FeedsTab extends React.Component { }; } - getAmendedFormData(): Feed | null { + getAmendedFormData(): Pick | null { if (this.formRef == null) { return null; } const formData = this.formRef.getFormData() as Partial; - if (formData.interval != null && formData.intervalMultiplier != null) { - formData.interval *= formData.intervalMultiplier; + const {url, label} = formData; + if (url == null || label == null) { + return null; } - delete formData.intervalMultiplier; + let {interval} = defaultFeed; + if (formData.interval != null && formData.intervalMultiplier != null) { + interval = formData.interval * formData.intervalMultiplier; + } - return {...defaultFeed, ...formData}; + return {url, label, interval}; } getIntervalSelectOptions() { @@ -207,10 +215,11 @@ class FeedsTab extends React.Component { ); } - getModifyFeedForm(feed: Feed) { - const isDayInterval = feed.interval % 1440; - const minutesDivisor = feed.interval % 60 ? 1 : 60; - const defaultIntervalTextValue = feed.interval / isDayInterval ? minutesDivisor : 1440; + getModifyFeedForm(feed: Partial) { + const feedInterval = feed.interval || defaultFeed.interval; + const isDayInterval = feedInterval % 1440; + const minutesDivisor = feedInterval % 60 ? 1 : 60; + const defaultIntervalTextValue = feedInterval / isDayInterval ? minutesDivisor : 1440; const defaultIntervalMultiplierId = isDayInterval ? minutesDivisor : 1440; return ( @@ -425,9 +434,9 @@ class FeedsTab extends React.Component { if (formData != null) { if (currentFeed === defaultFeed) { - FeedsStoreClass.addFeed(formData); + SettingsActions.addFeed(formData); } else if (currentFeed?._id != null) { - FeedsStoreClass.modifyFeed(currentFeed._id, formData); + SettingsActions.modifyFeed(currentFeed._id, formData); } } if (this.formRef != null) { @@ -451,7 +460,7 @@ class FeedsTab extends React.Component { handleRemoveFeedClick = (feed: Feed) => { if (feed._id != null) { - FeedsStoreClass.removeFeed(feed._id); + SettingsActions.removeFeedMonitor(feed._id); } if (feed === this.state.currentlyEditingFeed) { @@ -474,7 +483,7 @@ class FeedsTab extends React.Component { const feedBrowseForm = input.formData as {feedID: string; search: string}; if ((input.event.target as HTMLInputElement).type !== 'checkbox') { this.setState({selectedFeedID: feedBrowseForm.feedID}); - FeedsStoreClass.fetchItems({params: {id: feedBrowseForm.feedID, search: feedBrowseForm.search}}); + SettingsActions.fetchItems({id: feedBrowseForm.feedID, search: feedBrowseForm.search}); } }; @@ -485,11 +494,12 @@ class FeedsTab extends React.Component { const formData = this.manualAddingFormRef.getFormData(); - const downloadedTorrents = this.props.items - .filter((item, index) => formData[index]) - .map((torrent, index) => ({id: index, value: torrent.link})); + // TODO: Properly handle array of array of URLs + const torrentsToDownload = this.props.items + .filter((_item, index) => formData[index]) + .map((item, index) => ({id: index, value: item.torrentURLs[0]})); - UIActions.displayModal({id: 'add-torrents', initialURLs: downloadedTorrents}); + UIActions.displayModal({id: 'add-torrents', initialURLs: torrentsToDownload}); }; validateForm(): {errors?: FeedsTabStates['errors']; isValid: boolean} { diff --git a/client/src/javascript/components/sidebar/NotificationsButton.tsx b/client/src/javascript/components/sidebar/NotificationsButton.tsx index 7a47979d..62711034 100644 --- a/client/src/javascript/components/sidebar/NotificationsButton.tsx +++ b/client/src/javascript/components/sidebar/NotificationsButton.tsx @@ -1,5 +1,5 @@ import classnames from 'classnames'; -import {defineMessages, FormattedMessage, injectIntl, WrappedComponentProps} from 'react-intl'; +import {defineMessages, injectIntl, WrappedComponentProps} from 'react-intl'; import React from 'react'; import type {Notification, NotificationCount} from '@shared/types/Notification'; @@ -42,8 +42,11 @@ const MESSAGES = defineMessages({ 'notification.torrent.errored.body': { id: 'notification.torrent.errored.body', }, - 'notification.feed.downloaded.torrent.heading': { - id: 'notification.feed.downloaded.torrent.heading', + 'notification.feed.torrent.added.heading': { + id: 'notification.feed.torrent.added.heading', + }, + 'notification.feed.torrent.added.body': { + id: 'notification.feed.torrent.added.body', }, clearAll: { id: 'notification.clear.all', @@ -129,7 +132,7 @@ class NotificationsButton extends React.Component
  • - {`${newerFrom + 1} – ${newerTo}`} + {`${newerFrom + 1} - ${newerTo}`}
  • - {notification.data.ruleLabel} - {' / '} - {notification.data.feedLabel} - - ), - title: notification.data.title, - }} - /> - ); - } else { - const messageID = MESSAGES[`${notification.id}.body` as keyof typeof MESSAGES]; - notificationBody = intl.formatMessage(messageID, notification.data); - } - return (
  • - {intl.formatMessage(MESSAGES[`${notification.id}.heading` as keyof typeof MESSAGES])} + {intl.formatMessage( + MESSAGES[`${notification.id}.heading` as keyof typeof MESSAGES] || {id: 'general.error.unknown'}, + )} {' — '} {`${date} ${intl.formatMessage(MESSAGES.at)} ${time}`}
    -
    {notificationBody}
    +
    + {intl.formatMessage( + MESSAGES[`${notification.id}.body` as keyof typeof MESSAGES] || {id: 'general.error.unknown'}, + notification.data, + )} +
    • ); }; diff --git a/client/src/javascript/constants/ServerActions.ts b/client/src/javascript/constants/ServerActions.ts index 54c38dda..3682ccb0 100644 --- a/client/src/javascript/constants/ServerActions.ts +++ b/client/src/javascript/constants/ServerActions.ts @@ -1,13 +1,13 @@ import type {AuthAuthenticationResponse, AuthVerificationResponse} from '@shared/schema/api/auth'; import type {Credentials} from '@shared/schema/Auth'; import type {ClientSettings} from '@shared/types/ClientSettings'; +import type {Feed, Item, Rule} from '@shared/types/Feed'; import type {FloodSettings} from '@shared/types/FloodSettings'; import type {NotificationFetchOptions, NotificationState} from '@shared/types/Notification'; import type {ServerEvents} from '@shared/types/ServerEvents'; import type {TorrentDetails} from '@shared/types/Torrent'; import type {SettingsSaveOptions} from '../stores/SettingsStore'; -import type {Feeds, Items, Rules} from '../stores/FeedsStore'; type ErrorType = | 'AUTH_LOGIN_ERROR' @@ -182,22 +182,22 @@ interface SettingsFeedMonitorRemoveSuccessAction { interface SettingsFeedMonitorFeedsFetchSuccessAction { type: 'SETTINGS_FEED_MONITOR_FEEDS_FETCH_SUCCESS'; - data: Feeds; + data: Array; } interface SettingsFeedMonitorRulesFetchSuccessAction { type: 'SETTINGS_FEED_MONITOR_RULES_FETCH_SUCCESS'; - data: Rules; + data: Array; } interface SettingsFeedMonitorsFetchSuccessAction { type: 'SETTINGS_FEED_MONITORS_FETCH_SUCCESS'; - data: {feeds: Feeds; rules: Rules}; + data: {feeds: Array; rules: Array}; } interface SettingsFeedMonitorItemsFetchSuccessAction { type: 'SETTINGS_FEED_MONITOR_ITEMS_FETCH_SUCCESS'; - data: Items; + data: Array; } interface SettingsFetchRequestSuccessAction { diff --git a/client/src/javascript/i18n/strings.compiled.json b/client/src/javascript/i18n/strings.compiled.json index 3ac88c92..8a041908 100644 --- a/client/src/javascript/i18n/strings.compiled.json +++ b/client/src/javascript/i18n/strings.compiled.json @@ -979,6 +979,12 @@ "value": "Copy" } ], + "general.error.unknown": [ + { + "type": 0, + "value": "An unknown error occurred" + } + ], "general.of": [ { "type": 0, @@ -1021,6 +1027,18 @@ "value": "Clear All" } ], + "notification.feed.torrent.added.body": [ + { + "type": 1, + "value": "title" + } + ], + "notification.feed.torrent.added.heading": [ + { + "type": 0, + "value": "Feed Item Queued" + } + ], "notification.showing": [ { "type": 0, diff --git a/client/src/javascript/i18n/strings.json b/client/src/javascript/i18n/strings.json index 63204d44..627c67e2 100644 --- a/client/src/javascript/i18n/strings.json +++ b/client/src/javascript/i18n/strings.json @@ -115,9 +115,12 @@ "general.of": "of", "general.clipboard.copy": "Copy", "general.clipboard.copied": "Copied", + "general.error.unknown": "An unknown error occurred", "mediainfo.execError": "An error occurred while running mediainfo on the server. Check that mediainfo is installed and available in the PATH to Flood.", "mediainfo.fetching": "Fetching...", "mediainfo.heading": "Mediainfo Output", + "notification.feed.torrent.added.heading": "Feed Item Queued", + "notification.feed.torrent.added.body": "{title}", "notification.torrent.finished.heading": "Finished Downloading", "notification.torrent.finished.body": "{name}", "notification.torrent.errored.heading": "Error Reported", diff --git a/client/src/javascript/stores/FeedsStore.ts b/client/src/javascript/stores/FeedsStore.ts index 99960791..a46ff84e 100644 --- a/client/src/javascript/stores/FeedsStore.ts +++ b/client/src/javascript/stores/FeedsStore.ts @@ -1,171 +1,100 @@ +import type {Feed, Rule, Item} from '@shared/types/Feed'; + import AppDispatcher from '../dispatcher/AppDispatcher'; import SettingsActions from '../actions/SettingsActions'; import BaseStore from './BaseStore'; -export interface Feed { - _id?: string; - type?: 'feed'; - label: string; - url: string; - interval: number; - count?: number; -} +class FeedsStoreClass extends BaseStore { + private feeds: Array = []; + private rules: Array = []; + private items: Array = []; -export interface Rule { - _id?: string; - type?: 'rule'; - label: string; - feedID: string; - field?: string; - match: string; - exclude: string; - destination: string; - tags: Array; - startOnLoad: boolean; - isBasePath?: boolean; - count?: number; -} - -export interface Item { - title: string; - link: string; -} - -export type Feeds = Array; -export type Rules = Array; -export type Items = Array; - -export class FeedsStoreClass extends BaseStore { - feeds: Feeds = []; - rules: Rules = []; - items: Items = []; - - static addFeed(feed: Feed) { - SettingsActions.addFeed(feed); - } - - static modifyFeed(id: Feed['_id'], feed: Feed) { - SettingsActions.modifyFeed(id, feed); - } - - static addRule(rule: Rule) { - SettingsActions.addRule(rule); - } - - static fetchFeedMonitors() { - SettingsActions.fetchFeedMonitors(); - } - - static fetchFeeds(query: string) { - SettingsActions.fetchFeeds(query); - } - - static fetchItems(query: {params: {id: string; search: string}}) { - SettingsActions.fetchItems(query); - } - - static fetchRules(query: string) { - SettingsActions.fetchRules(query); - } - - static removeFeed(id: Feed['_id']) { - if (id != null) { - SettingsActions.removeFeedMonitor(id); - } - } - - static removeRule(id: Rule['_id']) { - if (id != null) { - SettingsActions.removeFeedMonitor(id); - } - } - - getFeeds() { + getFeeds(): Array { return this.feeds; } - getRules() { + getRules(): Array { return this.rules; } - getItems() { + getItems(): Array { return this.items; } - handleFeedAddError(error?: Error) { + handleFeedAddError(error?: Error): void { this.emit('SETTINGS_FEED_MONITOR_FEED_ADD_ERROR', error); } - handleFeedAddSuccess() { - FeedsStoreClass.fetchFeedMonitors(); + handleFeedAddSuccess(): void { + SettingsActions.fetchFeedMonitors(); this.emit('SETTINGS_FEED_MONITOR_FEED_ADD_SUCCESS'); } - handleFeedModifyError(error?: Error) { + handleFeedModifyError(error?: Error): void { this.emit('SETTINGS_FEED_MONITOR_FEED_MODIFY_ERROR', error); } - handleFeedModifySuccess() { - FeedsStoreClass.fetchFeedMonitors(); + handleFeedModifySuccess(): void { + SettingsActions.fetchFeedMonitors(); this.emit('SETTINGS_FEED_MONITOR_FEED_MODIFY_SUCCESS'); } - handleRuleAddError(error?: Error) { + handleRuleAddError(error?: Error): void { this.emit('SETTINGS_FEED_MONITOR_RULE_ADD_ERROR', error); } - handleRuleAddSuccess() { - FeedsStoreClass.fetchFeedMonitors(); + handleRuleAddSuccess(): void { + SettingsActions.fetchFeedMonitors(); this.emit('SETTINGS_FEED_MONITOR_RULE_ADD_SUCCESS'); } - handleFeedMonitorsFetchError(error?: Error) { + handleFeedMonitorsFetchError(error?: Error): void { this.emit('SETTINGS_FEED_MONITORS_FETCH_ERROR', error); } - handleFeedMonitorsFetchSuccess(feedMonitors: {feeds: Feeds; rules: Rules}) { + handleFeedMonitorsFetchSuccess(feedMonitors: {feeds: Array; rules: Array}): void { this.setFeeds(feedMonitors.feeds); this.setRules(feedMonitors.rules); this.emit('SETTINGS_FEED_MONITORS_FETCH_SUCCESS'); } - handleFeedMonitorRemoveError(id: string) { + handleFeedMonitorRemoveError(id: string): void { this.emit('SETTINGS_FEED_MONITOR_REMOVE_ERROR', id); } - handleFeedMonitorRemoveSuccess(id: string) { - FeedsStoreClass.fetchFeedMonitors(); + handleFeedMonitorRemoveSuccess(id: string): void { + SettingsActions.fetchFeedMonitors(); this.emit('SETTINGS_FEED_MONITOR_REMOVE_SUCCESS', id); } - handleFeedsFetchError(error?: Error) { + handleFeedsFetchError(error?: Error): void { this.emit('SETTINGS_FEED_MONITOR_FEEDS_FETCH_ERROR', error); } - handleFeedsFetchSuccess(feeds: Feeds) { + handleFeedsFetchSuccess(feeds: Array): void { this.setFeeds(feeds); this.emit('SETTINGS_FEED_MONITOR_FEEDS_FETCH_SUCCESS'); } - handleRulesFetchError(error?: Error) { + handleRulesFetchError(error?: Error): void { this.emit('SETTINGS_FEED_MONITOR_RULES_FETCH_ERROR', error); } - handleRulesFetchSuccess(rules: Rules) { + handleRulesFetchSuccess(rules: Array): void { this.setRules(rules); this.emit('SETTINGS_FEED_MONITOR_RULES_FETCH_SUCCESS'); } - handleItemsFetchError(error?: Error) { + handleItemsFetchError(error?: Error): void { this.emit('SETTINGS_FEED_MONITOR_ITEMS_FETCH_ERROR', error); } - handleItemsFetchSuccess(items: Items) { + handleItemsFetchSuccess(items: Array): void { this.items = items; this.emit('SETTINGS_FEED_MONITOR_ITEMS_FETCH_SUCCESS'); } - setFeeds(feeds: Feeds) { + setFeeds(feeds: Array): void { if (feeds == null) { this.feeds = []; return; @@ -176,7 +105,7 @@ export class FeedsStoreClass extends BaseStore { }); } - setRules(rules: Rules) { + setRules(rules: Array): void { if (rules == null) { this.rules = []; return; diff --git a/package-lock.json b/package-lock.json index b99d2baa..1b08afe4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "@types/debug": "^4.1.5", "@types/express": "^4.17.8", "@types/express-rate-limit": "^5.1.0", + "@types/feedsub": "^0.7.0", "@types/flux": "^3.1.9", "@types/fs-extra": "^9.0.2", "@types/geoip-country": "^4.0.0", @@ -2792,6 +2793,15 @@ "integrity": "sha1-jtIE2g9U6cjq7DGx7skeJRMtCCw=", "dev": true }, + "node_modules/@types/feedsub": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@types/feedsub/-/feedsub-0.7.0.tgz", + "integrity": "sha512-IXje246s7/rQrbn7KXpa8CRCO3eSl963s/iiyRwSxQNnnRHyTMtnSLxNbhbAhdpTccCSDGW28KwXhrq3pY7+7A==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/flux": { "version": "3.1.9", "resolved": "https://registry.npmjs.org/@types/flux/-/flux-3.1.9.tgz", @@ -30576,6 +30586,15 @@ "integrity": "sha1-jtIE2g9U6cjq7DGx7skeJRMtCCw=", "dev": true }, + "@types/feedsub": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@types/feedsub/-/feedsub-0.7.0.tgz", + "integrity": "sha512-IXje246s7/rQrbn7KXpa8CRCO3eSl963s/iiyRwSxQNnnRHyTMtnSLxNbhbAhdpTccCSDGW28KwXhrq3pY7+7A==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/flux": { "version": "3.1.9", "resolved": "https://registry.npmjs.org/@types/flux/-/flux-3.1.9.tgz", diff --git a/package.json b/package.json index e19dd208..aad24e46 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "@types/debug": "^4.1.5", "@types/express": "^4.17.8", "@types/express-rate-limit": "^5.1.0", + "@types/feedsub": "^0.7.0", "@types/flux": "^3.1.9", "@types/fs-extra": "^9.0.2", "@types/geoip-country": "^4.0.0", diff --git a/server/models/Feed.js b/server/models/Feed.js deleted file mode 100644 index 9181bd96..00000000 --- a/server/models/Feed.js +++ /dev/null @@ -1,75 +0,0 @@ -import FeedSub from 'feedsub'; - -// TODO: Use a type-checked Feed parser -class Feed { - constructor(options) { - this.options = options || {}; - this.options.maxItemHistory = options.maxItemHistory || 100; - this.items = []; - - if (!options.url) { - console.error('Feed URL must be defined.'); - return null; - } - - this.reader = new FeedSub(options.url, { - autoStart: true, - emitOnStart: true, - maxHistory: this.options.maxItemHistory, - interval: options.interval ? Number(options.interval) : 15, - forceInterval: true, - readEveryItem: true, - }); - - this.initReader(); - } - - modify(options) { - Object.assign(this.options, options); - this.items = []; - - this.reader = new FeedSub(options.url, { - autoStart: true, - emitOnStart: true, - maxHistory: this.options.maxItemHistory, - interval: options.interval ? Number(options.interval) : 15, - forceInterval: true, - readEveryItem: true, - }); - - this.initReader(); - } - - getItems() { - return this.items; - } - - handleFeedItems(items) { - const nextLength = this.items.length + items.length; - if (nextLength >= this.options.maxItemHistory) { - const diff = nextLength - this.options.maxHistory; - this.items = this.items.splice(0, diff); - } - - this.items = this.items.concat(items); - - this.options.onNewItems({ - feed: this.options, - items, - }); - } - - initReader() { - this.reader.on('items', this.handleFeedItems.bind(this)); - this.reader.on('error', (error) => { - console.log('Feed reader error:', error); - }); - this.reader.start(); - } - - stopReader() { - this.reader.stop(); - } -} - -export default Feed; diff --git a/server/models/FeedReader.ts b/server/models/FeedReader.ts new file mode 100644 index 00000000..61eb793e --- /dev/null +++ b/server/models/FeedReader.ts @@ -0,0 +1,74 @@ +import FeedSub, {FeedItem} from 'feedsub'; + +export interface FeedReaderOptions { + feedID: string; + feedLabel: string; + url: string; + interval: number; + maxHistory: number; + onNewItems: (options: FeedReaderOptions, items: Array) => void; +} + +class FeedReader { + private options: FeedReaderOptions; + private items: Array = []; + private reader: FeedSub | null = null; + + constructor(options: FeedReaderOptions) { + this.options = options; + + this.initReader(); + } + + modify(options: Partial) { + this.options = {...this.options, ...options}; + this.items = []; + + this.initReader(); + } + + getOptions() { + return this.options; + } + + getItems() { + return this.items; + } + + handleFeedItems(items: Array) { + const nextLength = this.items.length + items.length; + if (nextLength >= this.options.maxHistory) { + const diff = nextLength - this.options.maxHistory; + this.items = this.items.splice(0, diff); + } + + this.items = this.items.concat(items); + + this.options.onNewItems(this.options, items); + } + + initReader() { + this.reader = new FeedSub(this.options.url, { + autoStart: true, + emitOnStart: true, + maxHistory: this.options.maxHistory, + interval: this.options.interval, + forceInterval: true, + }); + + this.reader.on('items', this.handleFeedItems.bind(this)); + this.reader.on('error', (error) => { + console.log('Feed reader error:', error); + }); + this.reader.start(); + } + + stopReader() { + if (this.reader != null) { + this.reader.stop(); + this.reader = null; + } + } +} + +export default FeedReader; diff --git a/server/routes/api/feed-monitor.ts b/server/routes/api/feed-monitor.ts index 3912dfed..01fb0033 100644 --- a/server/routes/api/feed-monitor.ts +++ b/server/routes/api/feed-monitor.ts @@ -1,39 +1,184 @@ import express from 'express'; +import type {AddFeedOptions, AddRuleOptions, ModifyFeedOptions} from '@shared/types/api/feed-monitor'; + import ajaxUtil from '../../util/ajaxUtil'; const router = express.Router(); +/** + * GET /api/feed-monitor + * @summary Gets subscribed feeds and their automation rules + * @tags Feeds + * @security User + * @return {{feeds: Array; rules: Array}} 200 - success response - application/json + * @return {Error} 500 - failure response - application/json + */ router.get('/', (req, res) => { - req.services?.feedService.getAll(ajaxUtil.getResponseFn(res)); + const callback = ajaxUtil.getResponseFn(res); + + req.services?.feedService + .getAll() + .then((feedsAndRules) => { + callback(feedsAndRules); + }) + .catch((error) => { + callback(null, error); + }); }); -router.delete('/:id', (req, res) => { - req.services?.feedService.removeItem(req.params.id, ajaxUtil.getResponseFn(res)); +/** + * DELETE /api/feed-monitor/{id} + * @summary Deletes feed subscription or automation rule + * @tags Feeds + * @security User + * @param id.path - Unique ID of the item + * @return {} 200 - success response - application/json + * @return {Error} 500 - failure response - application/json + */ +router.delete<{id: string}>('/:id', (req, res) => { + const callback = ajaxUtil.getResponseFn(res); + + req.services?.feedService + .removeItem(req.params.id) + .then(() => { + callback(null); + }) + .catch((error) => { + callback(null, error); + }); }); +/** + * GET /api/feed-monitor/feeds + * @summary Gets subscribed feeds + * @tags Feeds + * @security User + * @return {Array}} 200 - success response - application/json + * @return {Error} 500 - failure response - application/json + */ router.get('/feeds', (req, res) => { - req.services?.feedService.getFeeds(req.params.query, ajaxUtil.getResponseFn(res)); + const callback = ajaxUtil.getResponseFn(res); + + req.services?.feedService + .getFeeds() + .then((feeds) => { + callback(feeds); + }) + .catch((error) => { + callback(null, error); + }); }); -router.put('/feeds', (req, res) => { - req.services?.feedService.addFeed(req.body, ajaxUtil.getResponseFn(res)); +/** + * PUT /api/feed-monitor/feeds + * @summary Subscribes to a feed + * @tags Feeds + * @security User + * @param {AddFeedOptions} request.body.required - options - application/json + * @return {Feed} 200 - success response - application/json + * @return {Error} 500 - failure response - application/json + */ +router.put('/feeds', (req, res) => { + const callback = ajaxUtil.getResponseFn(res); + + req.services?.feedService + .addFeed(req.body) + .then((feed) => { + callback(feed); + }) + .catch((error) => { + callback(null, error); + }); }); -router.put('/feeds/:id', (req, res) => { - req.services?.feedService.modifyFeed(req.params.id, req.body, ajaxUtil.getResponseFn(res)); +/** + * PUT /api/feed-monitor/feeds/{id} + * @summary Modifies the options of a feed subscription + * @tags Feeds + * @security User + * @param id.path - Unique ID of the feed subscription + * @param {ModifyFeedOptions} request.body.required - options - application/json + * @return {} 200 - success response - application/json + * @return {Error} 500 - failure response - application/json + */ +router.put<{id: string}, unknown, ModifyFeedOptions>('/feeds/:id', (req, res) => { + const callback = ajaxUtil.getResponseFn(res); + + req.services?.feedService + .modifyFeed(req.params.id, req.body) + .then(() => { + callback(null); + }) + .catch((error) => { + callback(null, error); + }); }); +/** + * GET /api/feed-monitor/feeds/{id}/items?search= + * @summary Gets items in a feed + * @tags Feeds + * @security User + * @param id.path - Unique ID of the feed subscription + * @param {string} search.query - string to search in items + * @return {Array} 200 - success response - application/json + * @return {Error} 500 - failure response - application/json + */ +router.get<{id: string}, unknown, ModifyFeedOptions, {search: string}>('/feeds/:id/items', (req, res) => { + const callback = ajaxUtil.getResponseFn(res); + + req.services?.feedService + .getItems(req.params.id, req.query.search) + .then((items) => { + callback(items); + }) + .catch((error) => { + callback(null, error); + }); +}); + +/** + * GET /api/feed-monitor/rules + * @summary Gets automation rules + * @tags Feeds + * @security User + * @return {Array}} 200 - success response - application/json + * @return {Error} 500 - failure response - application/json + */ router.get('/rules', (req, res) => { - req.services?.feedService.getRules(req.params.query, ajaxUtil.getResponseFn(res)); + const callback = ajaxUtil.getResponseFn(res); + + req.services?.feedService + .getRules() + .then((rules) => { + callback(rules); + }) + .catch((error) => { + callback(null, error); + }); }); -router.put('/rules', (req, res) => { - req.services?.feedService.addRule(req.body, ajaxUtil.getResponseFn(res)); -}); +/** + * PUT /api/feed-monitor/rules + * @summary Adds an automation rule to a feed subscription + * @tags Feeds + * @security User + * @param {AddRuleOptions} request.body.required - options - application/json + * @return {Rule} 200 - success response - application/json + * @return {Error} 500 - failure response - application/json + */ +router.put('/rules', (req, res) => { + const callback = ajaxUtil.getResponseFn(res); -router.get('/items', (req, res) => { - req.services?.feedService.getItems(req.query, ajaxUtil.getResponseFn(res)); + req.services?.feedService + .addRule(req.body) + .then(() => { + callback(null); + }) + .catch((error) => { + callback(null, error); + }); }); export default router; diff --git a/server/services/feedService.js b/server/services/feedService.js deleted file mode 100644 index cadcb875..00000000 --- a/server/services/feedService.js +++ /dev/null @@ -1,365 +0,0 @@ -import path from 'path'; -import Datastore from 'nedb'; - -import BaseService from './BaseService'; -import config from '../../config'; -import Feed from '../models/Feed'; -import regEx from '../../shared/util/regEx'; - -// TODO: Allow users to specify which key contains the URLs. -const getTorrentUrlsFromItem = (feedItem) => { - // If we've got an Array of enclosures, we'll iterate over the values and - // look for the url key. - if (feedItem.enclosures && Array.isArray(feedItem.enclosures)) { - return feedItem.enclosures.reduce((urls, enclosure) => { - if (enclosure.url) { - urls.push(enclosure.url); - } - - return urls; - }, []); - } - - // If we've got a Object of enclosures, use url key - if (feedItem.enclosure && feedItem.enclosure.url) { - return [feedItem.enclosure.url]; - } - - // If there are no enclosures, then use the link tag instead - if (feedItem.link) { - // remove CDATA tags around links - const cdata = regEx.cdata.exec(feedItem.link); - - if (cdata && cdata[1]) { - return [cdata[1]]; - } - - return [feedItem.link]; - } - - return []; -}; - -const getItemsMatchingRules = (feedItems, rules, feed) => { - return feedItems.reduce((matchedItems, feedItem) => { - rules.forEach((rule) => { - const isMatched = new RegExp(rule.match, 'gi').test(feedItem[rule.field]); - const isExcluded = rule.exclude !== '' && new RegExp(rule.exclude, 'gi').test(feedItem[rule.field]); - - if (isMatched && !isExcluded) { - const torrentUrls = getTorrentUrlsFromItem(feedItem); - const isAlreadyDownloaded = matchedItems.some((matchedItem) => - torrentUrls.every((url) => matchedItem.urls.includes(url)), - ); - - if (!isAlreadyDownloaded) { - matchedItems.push({ - urls: torrentUrls, - tags: rule.tags, - feedID: rule.feedID, - feedLabel: feed.label, - matchTitle: feedItem.title, - ruleID: rule._id, - ruleLabel: rule.label, - destination: rule.destination, - startOnLoad: rule.startOnLoad, - }); - } - } - }); - - return matchedItems; - }, []); -}; - -const getUrlsFromItems = (feedItems) => { - return feedItems.reduce((urls, feedItem) => urls.concat(feedItem.urls), []); -}; - -class FeedService extends BaseService { - constructor(...args) { - super(...args); - - this.db = this.loadDatabase(); - this.onServicesUpdated = () => { - this.init(); - }; - } - - addFeed(feed, callback) { - this.addItem('feed', feed, (newFeed) => { - this.startNewFeed(newFeed); - callback(newFeed); - }); - } - - modifyFeed(id, feedToModify, callback) { - const modifiedFeed = this.feeds.find((feed) => feed.options._id === id); - modifiedFeed.stopReader(); - modifiedFeed.modify(feedToModify); - this.modifyItem(id, feedToModify, (err) => { - callback(err); - }); - } - - addItem(type, item, callback) { - if (this.db == null) { - return; - } - - this.db.insert(Object.assign(item, {type}), (err, newDoc) => { - if (err) { - callback(null, err); - return; - } - - callback(newDoc); - }); - } - - modifyItem(id, newItem, callback) { - if (this.db == null) { - return; - } - - this.db.update({_id: id}, {$set: newItem}, {}, (err) => { - if (err) { - callback(null, err); - return; - } - - callback(null); - }); - } - - addRule(rule, callback) { - this.addItem('rule', rule, (newRule, error) => { - if (error) { - callback(null, error); - return; - } - - callback(newRule); - - if (this.rules[newRule.feedID] == null) { - this.rules[newRule.feedID] = []; - } - - this.rules[newRule.feedID].push(newRule); - - const associatedFeed = this.feeds.find((feed) => feed.options._id === newRule.feedID); - - if (associatedFeed) { - this.handleNewItems({ - feed: associatedFeed.options, - items: associatedFeed.getItems(), - }); - } - }); - } - - getAll(callback) { - this.db.find({}, (err, docs) => { - if (err) { - callback(null, err); - return; - } - - callback( - docs.reduce((memo, item) => { - const type = `${item.type}s`; - - if (memo[type] == null) { - memo[type] = []; - } - - memo[type].push(item); - - return memo; - }, {}), - ); - }); - } - - getFeeds(query, callback) { - this.queryItem('feed', query, callback); - } - - getItems(query, callback) { - const selectedFeed = this.feeds.find((feed) => feed.options._id === query.id); - - if (selectedFeed) { - const items = selectedFeed.getItems(); - - if (query.search) { - callback(items.filter((item) => item.title.toLowerCase().indexOf(query.search.toLowerCase()) !== -1)); - } else { - callback(items); - } - } else { - callback(null); - } - } - - getPreviouslyMatchedUrls() { - return new Promise((resolve, reject) => { - this.db.find({type: 'matchedTorrents'}, (err, docs) => { - if (err) { - reject(err); - } - - resolve(docs.reduce((matchedUrls, doc) => matchedUrls.concat(doc.urls), [])); - }); - }); - } - - getRules(query, callback) { - this.queryItem('rule', query, callback); - } - - handleNewItems({items: feedItems, feed}) { - this.getPreviouslyMatchedUrls() - .then((previouslyMatchedUrls) => { - const applicableRules = this.rules[feed._id]; - if (!applicableRules) return; - - const itemsMatchingRules = getItemsMatchingRules(feedItems, applicableRules, feed); - const itemsToDownload = itemsMatchingRules.filter((item) => - item.urls.some((url) => !previouslyMatchedUrls.includes(url)), - ); - - const lastAddUrlCallback = () => { - const urlsToAdd = getUrlsFromItems(itemsToDownload); - - this.db.update({type: 'matchedTorrents'}, {$push: {urls: {$each: urlsToAdd}}}, {upsert: true}); - - this.services.notificationService.addNotification( - itemsToDownload.map((item) => ({ - id: 'notification.feed.downloaded.torrent', - data: { - feedLabel: item.feedLabel, - ruleLabel: item.ruleLabel, - title: item.matchTitle, - }, - })), - ); - this.services.torrentService.fetchTorrentList(); - }; - - itemsToDownload.forEach((item, index) => { - this.services.clientGatewayService - .addTorrentsByURL({ - urls: item.urls, - destination: item.destination, - isBasePath: false, - start: item.startOnLoad, - tags: item.tags, - }) - .then(() => { - if (index === itemsToDownload.length - 1) { - lastAddUrlCallback(); - } - - this.db.update({_id: item.ruleID}, {$inc: {count: 1}}, {upsert: true}); - }) - .catch(console.error); - }); - }) - .catch(console.error); - } - - init() { - this.feeds = []; - this.rules = {}; - this.db.find({}, (err, docs) => { - if (err) { - return; - } - - // Create two arrays, one for feeds and one for rules. - const feedsSummary = docs.reduce( - (accumulator, doc) => { - if (doc.type === 'feed' || doc.type === 'rule') { - accumulator[`${doc.type}s`].push(doc); - } - - return accumulator; - }, - {feeds: [], rules: []}, - ); - - // Add all download rules to the local state. - feedsSummary.rules.forEach((rule) => { - if (this.rules[rule.feedID] == null) { - this.rules[rule.feedID] = []; - } - - this.rules[rule.feedID].push(rule); - }); - - // Initiate all feeds. - feedsSummary.feeds.forEach((feed) => { - this.startNewFeed(feed); - }); - }); - } - - loadDatabase() { - if (this.db != null) { - return this.db; - } - - const db = new Datastore({ - autoload: true, - filename: path.join(config.dbPath, this.user._id, 'settings', 'feeds.db'), - }); - - return db; - } - - queryItem(type, query, callback) { - query = query || {}; - - this.db.find(Object.assign(query, {type}), (err, docs) => { - if (err) { - callback(null, err); - return; - } - - callback(docs); - }); - } - - removeItem(id, callback) { - let indexToRemove = -1; - const itemToRemove = this.feeds.find((feed, index) => { - if (feed.options._id === id) { - indexToRemove = index; - return true; - } - - return false; - }); - - if (itemToRemove != null) { - itemToRemove.stopReader(); - this.feeds.splice(indexToRemove, 1); - } - - this.db.remove({_id: id}, {}, (err, docs) => { - if (err) { - callback(null, err); - return; - } - - callback(docs); - }); - } - - startNewFeed(feedConfig) { - feedConfig.onNewItems = this.handleNewItems.bind(this); - this.feeds.push(new Feed(feedConfig)); - } -} - -export default FeedService; diff --git a/server/services/feedService.ts b/server/services/feedService.ts new file mode 100644 index 00000000..790ea207 --- /dev/null +++ b/server/services/feedService.ts @@ -0,0 +1,386 @@ +import path from 'path'; +import Datastore from 'nedb'; + +import type {FeedItem} from 'feedsub'; + +import BaseService from './BaseService'; +import config from '../../config'; +import FeedReader from '../models/FeedReader'; +import {getFeedItemsMatchingRules, getTorrentUrlsFromFeedItem} from '../util/feedUtil'; + +import type {AddFeedOptions, AddRuleOptions, ModifyFeedOptions} from '../../shared/types/api/feed-monitor'; +import type {Feed, Item, MatchedTorrents, Rule} from '../../shared/types/Feed'; +import type {FeedReaderOptions} from '../models/FeedReader'; + +class FeedService extends BaseService { + db = this.loadDatabase(); + feedReaders: Array = []; + rules: { + [feedID: string]: Array; + } = {}; + + constructor(...args: ConstructorParameters) { + super(...args); + + this.onServicesUpdated = () => { + this.init(); + }; + } + + /** + * Subscribes to a feed + * + * @param {AddFeedOptions} options - An object of options... + * @return {Promise} - Resolves with Feed or rejects with error. + */ + async addFeed({url, label, interval}: AddFeedOptions): Promise { + if (typeof url !== 'string' || typeof label !== 'string' || typeof interval !== 'number') { + throw new Error('Unprocessable Entity'); + } + + if (this.db == null) { + throw new Error(''); + } + + const newFeed = await new Promise((resolve, reject) => { + this.db.insert({type: 'feed', url, label, interval}, (err, newDoc) => { + if (err) { + reject(err); + return; + } + + resolve(newDoc as Feed); + }); + }); + + this.startNewFeed(newFeed); + + return newFeed; + } + + /** + * Modifies the options of a feed subscription + * + * @param {string} id - Unique ID of the feed + * @param {ModifyFeedOptions} options - An object of options... + * @return {Promise} - Rejects with error. + */ + async modifyFeed(id: string, {url, label, interval}: ModifyFeedOptions): Promise { + if (url != null && typeof url !== 'string') { + throw new Error(); + } + + if (label != null && typeof label !== 'string') { + throw new Error(); + } + + if (interval != null && typeof interval !== 'number') { + throw new Error(); + } + + const modifiedFeedReader = this.feedReaders.find((feedReader) => feedReader.getOptions().feedID === id); + + if (modifiedFeedReader == null || this.db == null) { + throw new Error(); + } + + modifiedFeedReader.stopReader(); + modifiedFeedReader.modify({feedLabel: label, url, interval}); + + return new Promise((resolve, reject) => { + this.db.update({_id: id}, {$set: {url, label, interval}}, {}, (err) => { + if (err) { + reject(err); + return; + } + + resolve(); + }); + }); + } + + async addRule(options: AddRuleOptions): Promise { + const newRule = await new Promise((resolve, reject) => { + this.db.insert({type: 'rule', ...options}, (err, newDoc) => { + if (err) { + reject(err); + return; + } + + resolve(newDoc as Rule); + }); + }); + + if (this.rules[newRule.feedID] == null) { + this.rules[newRule.feedID] = []; + } + + this.rules[newRule.feedID].push(newRule); + + const associatedFeedReader = this.feedReaders.find( + (feedReader) => feedReader.getOptions().feedID === newRule.feedID, + ); + + if (associatedFeedReader) { + this.handleNewItems(associatedFeedReader.getOptions(), associatedFeedReader.getItems()); + } + + return newRule; + } + + async getAll(): Promise<{feeds: Array; rules: Array}> { + if (this.db == null) { + throw new Error(); + } + + return new Promise<{feeds: Array; rules: Array}>((resolve, reject) => { + this.db.find({}, (err: Error | null, docs: Array) => { + if (err) { + reject(err); + return; + } + + resolve( + docs.reduce( + (memo: {feeds: Array; rules: Array}, item) => { + if (item.type === 'feed') { + memo.feeds.push(item); + } + + if (item.type === 'rule') { + memo.rules.push(item); + } + + return memo; + }, + {feeds: [], rules: []}, + ), + ); + }); + }); + } + + async getFeeds(): Promise> { + return new Promise>((resolve, reject) => { + this.db.find({type: 'feed'}, (err: Error | null, feeds: Array) => { + if (err) { + reject(err); + return; + } + + resolve(feeds); + }); + }); + } + + async getItems(id: string, search: string): Promise> { + const selectedFeedReader = this.feedReaders.find((feedReader) => feedReader.getOptions().feedID === id); + + if (selectedFeedReader == null) { + throw new Error(); + } + + const items = selectedFeedReader.getItems(); + + const filteredItems = search + ? items.filter((item) => { + if (typeof item.title === 'string') { + return item.title.toLowerCase().includes(search.toLowerCase()); + } + return false; + }) + : items; + + return filteredItems.map((item) => { + return { + title: typeof item.title === 'string' ? item.title : 'Unknown', + torrentURLs: getTorrentUrlsFromFeedItem(item), + }; + }); + } + + async getPreviouslyMatchedUrls(): Promise> { + return new Promise((resolve, reject) => { + this.db.find({type: 'matchedTorrents'}, (err: Error, docs: Array) => { + if (err) { + reject(err); + return; + } + + resolve(docs.reduce((matchedUrls: Array, doc) => matchedUrls.concat(doc.urls), [])); + }); + }); + } + + async getRules(): Promise> { + return new Promise>((resolve, reject) => { + this.db.find({type: 'rule'}, (err: Error | null, rules: Array) => { + if (err) { + reject(err); + return; + } + + resolve(rules); + }); + }); + } + + handleNewItems(feedReaderOptions: FeedReaderOptions, feedItems: Array): void { + this.getPreviouslyMatchedUrls() + .then((previouslyMatchedUrls) => { + const {feedID, feedLabel} = feedReaderOptions; + const applicableRules = this.rules[feedID]; + if (!applicableRules) return; + + const itemsMatchingRules = getFeedItemsMatchingRules(feedItems, applicableRules); + const itemsToDownload = itemsMatchingRules.filter((item) => + item.urls.some((url) => !previouslyMatchedUrls.includes(url)), + ); + + if (itemsToDownload.length === 0) { + return; + } + + Promise.all( + itemsToDownload.map( + async (item): Promise> => { + const {urls, destination, start, tags, ruleID} = item; + await this?.services?.clientGatewayService + ?.addTorrentsByURL({ + urls, + destination, + start, + tags, + }) + .then(() => { + this.db.update({_id: feedID}, {$inc: {count: 1}}, {upsert: true}); + this.db.update({_id: ruleID}, {$inc: {count: 1}}, {upsert: true}); + }) + .catch(console.error); + + return urls; + }, + ), + ).then((ArrayOfURLArrays) => { + const addedURLs = ArrayOfURLArrays.reduce( + (URLArray: Array, urls: Array) => URLArray.concat(urls), + [], + ); + + this.db.update({type: 'matchedTorrents'}, {$push: {urls: {$each: addedURLs}}}, {upsert: true}); + + this.services?.notificationService.addNotification( + itemsToDownload.map((item) => ({ + id: 'notification.feed.torrent.added', + data: { + title: item.matchTitle, + feedLabel, + ruleLabel: item.ruleLabel, + }, + })), + ); + this.services?.torrentService.fetchTorrentList(); + }); + }) + .catch(console.error); + } + + init() { + this.db.find({}, (err: Error, docs: Array) => { + if (err) { + return; + } + + // Create two arrays, one for feeds and one for rules. + const feedsSummary: {feeds: Array; rules: Array} = docs.reduce( + (accumulator: {feeds: Array; rules: Array}, doc) => { + if (doc.type === 'feed') { + accumulator.feeds.push(doc); + } + + if (doc.type === 'rule') { + accumulator.rules.push(doc); + } + + return accumulator; + }, + {feeds: [], rules: []}, + ); + + // Add all download rules to the local state. + feedsSummary.rules.forEach((rule) => { + if (this.rules[rule.feedID] == null) { + this.rules[rule.feedID] = []; + } + + this.rules[rule.feedID].push(rule); + }); + + // Initiate all feeds. + feedsSummary.feeds.forEach((feed) => { + this.startNewFeed(feed); + }); + }); + } + + loadDatabase(): Datastore { + if (this.db != null) { + return this.db; + } + + const db = new Datastore({ + autoload: true, + filename: path.join(config.dbPath, this.user._id, 'settings', 'feeds.db'), + }); + + return db; + } + + async removeItem(id: string): Promise { + let feedReaderToRemoveIndex = -1; + const feedReaderToRemove = this.feedReaders.find((feedReader, index) => { + if (feedReader.getOptions().feedID === id) { + feedReaderToRemoveIndex = index; + return true; + } + + return false; + }); + + if (feedReaderToRemove != null) { + feedReaderToRemove.stopReader(); + this.feedReaders.splice(feedReaderToRemoveIndex, 1); + } + + return new Promise((resolve, reject) => { + this.db.remove({_id: id}, {}, (err) => { + if (err) { + reject(err); + return; + } + + resolve(); + }); + }); + } + + startNewFeed(feed: Feed) { + const {_id: feedID, label: feedLabel, url, interval} = feed; + + if (typeof feedID !== 'string' || typeof url !== 'string') { + return false; + } + + if (typeof interval !== 'number') { + return false; + } + + this.feedReaders.push( + new FeedReader({feedID, feedLabel, url, interval, maxHistory: 100, onNewItems: this.handleNewItems.bind(this)}), + ); + + return true; + } +} + +export default FeedService; diff --git a/server/services/notificationService.ts b/server/services/notificationService.ts index 41b61f40..e9d4a333 100644 --- a/server/services/notificationService.ts +++ b/server/services/notificationService.ts @@ -37,7 +37,7 @@ class NotificationService extends BaseService { data: notification.data, id: notification.id, read: false, - })); + })) as Notification[]; this.db.insert(notificationsToInsert, () => this.emitUpdate()); } diff --git a/server/util/feedUtil.ts b/server/util/feedUtil.ts new file mode 100644 index 00000000..5e2f1c61 --- /dev/null +++ b/server/util/feedUtil.ts @@ -0,0 +1,88 @@ +import type {FeedItem} from 'feedsub'; + +import regEx from '../../shared/util/regEx'; + +import type {AddTorrentByURLOptions} from '../../shared/types/api/torrents'; +import type {Rule} from '../../shared/types/Feed'; + +interface PendingDownloadItems extends AddTorrentByURLOptions { + matchTitle: string; + ruleID: string; + ruleLabel: string; +} + +interface RSSEnclosure { + url?: string; + length?: number; + type?: string; +} + +// TODO: Allow users to specify which key contains the URLs. +export const getTorrentUrlsFromFeedItem = (feedItem: FeedItem): Array => { + // If we've got an Array of enclosures, we'll iterate over the values and + // look for the url key. + const RSSEnclosures = feedItem.enclosures as Array | undefined; + if (RSSEnclosures && Array.isArray(RSSEnclosures)) { + return RSSEnclosures.reduce((urls: Array, enclosure) => { + if (enclosure.url) { + urls.push(enclosure.url); + } + + return urls; + }, []); + } + + // If we've got a Object of enclosures, use url key + const RSSEnclosure = feedItem.enclosure as RSSEnclosure | undefined; + if (RSSEnclosure?.url) { + return [RSSEnclosure.url]; + } + + // If there are no enclosures, then use the link tag instead + if (feedItem.link) { + // remove CDATA tags around links + const cdata = regEx.cdata.exec(feedItem.link as string); + + if (cdata && cdata[1]) { + return [cdata[1]]; + } + + return [feedItem.link as string]; + } + + return []; +}; + +export const getFeedItemsMatchingRules = ( + feedItems: Array, + rules: Array, +): Array => { + return feedItems.reduce((matchedItems: Array, feedItem) => { + rules.forEach((rule) => { + const matchField = rule.field ? (feedItem[rule.field] as string) : (feedItem.title as string); + const isMatched = new RegExp(rule.match, 'gi').test(matchField); + const isExcluded = rule.exclude !== '' && new RegExp(rule.exclude, 'gi').test(matchField); + + if (isMatched && !isExcluded) { + const torrentUrls = getTorrentUrlsFromFeedItem(feedItem); + const isAlreadyDownloaded = matchedItems.some((matchedItem) => + torrentUrls.every((url) => matchedItem.urls.includes(url)), + ); + + if (!isAlreadyDownloaded) { + matchedItems.push({ + urls: torrentUrls, + tags: rule.tags, + matchTitle: feedItem.title as string, + ruleID: rule._id, + ruleLabel: rule.label, + destination: rule.destination, + start: rule.startOnLoad, + }); + } + } + }); + + return matchedItems; + }, []); +}; diff --git a/shared/types/Feed.ts b/shared/types/Feed.ts new file mode 100644 index 00000000..e8a9e4db --- /dev/null +++ b/shared/types/Feed.ts @@ -0,0 +1,34 @@ +export interface Feed { + type: 'feed'; + _id: string; + label: string; + url: string; + interval: number; + count?: number; +} + +export interface Rule { + type: 'rule'; + _id: string; + label: string; + feedID: string; + field?: string; + match: string; + exclude: string; + destination: string; + tags: Array; + startOnLoad: boolean; + isBasePath?: boolean; + count?: number; +} + +export interface MatchedTorrents { + type: 'matchedTorrents'; + _id: string; + urls: Array; +} + +export interface Item { + title: string; + torrentURLs: Array; +} diff --git a/shared/types/Notification.ts b/shared/types/Notification.ts index 46d1c5ed..639d86fe 100644 --- a/shared/types/Notification.ts +++ b/shared/types/Notification.ts @@ -1,16 +1,27 @@ -export interface Notification { +export interface BaseNotification { _id?: string; - id: 'notification.torrent.finished' | 'notification.torrent.errored' | 'notification.feed.downloaded.torrent'; read: boolean; ts: number; // timestamp +} + +export interface TorrentNotification extends BaseNotification { + id: 'notification.torrent.finished' | 'notification.torrent.errored'; data: { name: string; - ruleLabel?: string; - feedLabel?: string; - title?: string; }; } +export interface FeedNotification extends BaseNotification { + id: 'notification.feed.torrent.added'; + data: { + title: string; + feedLabel: string; + ruleLabel: string; + }; +} + +export type Notification = TorrentNotification | FeedNotification; + export interface NotificationCount { total: number; unread: number; diff --git a/shared/types/api/feed-monitor.ts b/shared/types/api/feed-monitor.ts new file mode 100644 index 00000000..890b2c5a --- /dev/null +++ b/shared/types/api/feed-monitor.ts @@ -0,0 +1,7 @@ +import type {Feed, Rule} from '../Feed'; + +export type AddFeedOptions = Omit; + +export type ModifyFeedOptions = Partial; + +export type AddRuleOptions = Omit;