zoriya/flood
1
0
Fork 0
mirror of https://github.com/zoriya/flood.git synced 2025-12-05 23:06:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add custom scripts for rss download

Browse Source
This commit is contained in:
Zoe Roux
2023-10-08 22:33:44 +02:00
parent b32abb6615
commit 875ae36d17
7 changed files with 102 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
client/src/javascript/components/modals/feeds-modal/DownloadRuleForm.tsx
View File

@@ -101,6 +101,9 @@ const DownloadRuleForm: FC<DownloadRuleFormProps> = ({
)}
</Textbox>
</FormRow>
<FormRow>
<Textbox id="script" label={i18n._('feeds.script')} defaultValue={rule.script} />
</FormRow>
<FormRow>
<FormRowItem>
<FilesystemBrowserTextbox

10
client/src/javascript/components/modals/feeds-modal/DownloadRulesTab.tsx
View File

@@ -17,6 +17,7 @@ const initialRule: AddRuleOptions = {
feedIDs: [],
match: '',
exclude: '',
script: '',
tags: [],
destination: '',
startOnLoad: false,
@@ -36,7 +37,12 @@ const validatedFields = {
error: 'feeds.validation.must.specify.label',
},
match: {
isValid: (value: string | undefined) => isNotEmpty(value) && isRegExValid(value),
isValid: (value: string | undefined) => {
if (isNotEmpty(value)) {
return isRegExValid(value);
}
return true;
},
error: 'feeds.validation.invalid.regular.expression',
},
exclude: {
@@ -64,6 +70,7 @@ interface RuleFormData {
feedID: string;
label: string;
match: string;
script: string;
tags: string;
isBasePath: boolean;
startOnLoad: boolean;
@@ -140,6 +147,7 @@ const DownloadRulesTab: FC = () => {
field: formData.field,
match: formData.match ?? initialRule.match,
exclude: formData.exclude ?? initialRule.exclude,
script: formData.script ?? initialRule.script,
destination: formData.destination ?? initialRule.destination,
tags: formData.tags?.split(',') ?? initialRule.tags,
startOnLoad: formData.startOnLoad ?? initialRule.startOnLoad,

1
client/src/javascript/i18n/strings/en.json
View File

@@ -102,6 +102,7 @@
"feeds.no.items.matching": "No items matching search term.",
"feeds.no.rules.defined": "No rules defined.",
"feeds.regEx": "RegEx",
"feeds.script": "Script",
"feeds.search": "Search term",
"feeds.search.term": "Search term",
"feeds.select.feed": "Select Feed",

1
server/routes/api/feed-monitor.test.ts
View File

@@ -220,6 +220,7 @@ describe('PUT /api/feed-monitor/rules', () => {
feedIDs: [''],
match: '',
exclude: '.*',
script: '',
destination: tempDirectory,
tags: ['FeedItem'],
startOnLoad: false,

68
server/services/feedService.ts
View File

@@ -70,6 +70,7 @@ class FeedService extends BaseService<Record<string, never>> {
field: rule.field,
match: rule.match,
exclude: rule.exclude,
script: rule.script,
startOnLoad: rule.startOnLoad,
isBasePath: rule.isBasePath,
});
@@ -259,22 +260,22 @@ class FeedService extends BaseService<Record<string, never>> {
}
handleNewItems = (feedReaderOptions: FeedReaderOptions, feedItems: Array<FeedItem>): void => {
this.getPreviouslyMatchedUrls()
.then((previouslyMatchedUrls) => {
const {feedID, feedLabel} = feedReaderOptions;
const applicableRules = this.rules[feedID];
if (!applicableRules) return;
this.getPreviouslyMatchedUrls().then(async (previouslyMatchedUrls) => {
const {feedID, feedLabel} = feedReaderOptions;
const applicableRules = this.rules[feedID];
if (!applicableRules) return;
const itemsMatchingRules = getFeedItemsMatchingRules(feedItems, applicableRules);
const itemsToDownload = itemsMatchingRules.filter((item) =>
item.urls.some((url) => !previouslyMatchedUrls.includes(url)),
);
const itemsMatchingRules = await getFeedItemsMatchingRules(feedItems, applicableRules);
const itemsToDownload = itemsMatchingRules.filter((item) =>
item.urls.some((url) => !previouslyMatchedUrls.includes(url)),
);
if (itemsToDownload.length === 0) {
return;
}
if (itemsToDownload.length === 0) {
return;
}
Promise.all(
try {
const ArrayOfURLArrays = await Promise.all(
itemsToDownload.map(async (item): Promise<Array<string>> => {
const {urls, destination, start, tags, ruleID} = item;
@@ -298,28 +299,29 @@ class FeedService extends BaseService<Record<string, never>> {
return urls;
}),
).then((ArrayOfURLArrays) => {
const addedURLs = ArrayOfURLArrays.reduce(
(URLArray: Array<string>, urls: Array<string>) => URLArray.concat(urls),
[],
);
);
const addedURLs = ArrayOfURLArrays.reduce(
(URLArray: Array<string>, urls: Array<string>) => URLArray.concat(urls),
[],
);
this.db.update({type: 'matchedTorrents'}, {$push: {urls: {$each: addedURLs}}}, {upsert: true});
this.db.update({type: 'matchedTorrents'}, {$push: {urls: {$each: addedURLs}}}, {upsert: true});
this.services?.notificationService.addNotification(
itemsToDownload.map((item) => ({
id: 'notification.feed.torrent.added',
data: {
title: item.matchTitle,
feedLabel,
ruleLabel: item.ruleLabel,
},
})),
);
this.services?.torrentService.fetchTorrentList();
});
})
.catch(console.error);
this.services?.notificationService.addNotification(
itemsToDownload.map((item) => ({
id: 'notification.feed.torrent.added',
data: {
title: item.matchTitle,
feedLabel,
ruleLabel: item.ruleLabel,
},
})),
);
this.services?.torrentService.fetchTorrentList();
} catch (e) {
console.error(e);
}
});
};
async removeItem(id: string): Promise<void> {

78
server/util/feedUtil.ts
View File

@@ -1,3 +1,5 @@
import {spawn} from 'node:child_process';
import type {FeedItem} from 'feedsub';
import type {AddTorrentByURLOptions} from '../../shared/schema/api/torrents';
@@ -53,36 +55,58 @@ export const getTorrentUrlsFromFeedItem = (feedItem: FeedItem): Array<string> =>
return [];
};
export const getFeedItemsMatchingRules = (
const execAsync = (...command: string[]) => {
const p = spawn(command[0], command.slice(1));
return new Promise((resolveFunc) => {
p.stdout.on('data', (x) => {
process.stdout.write(x.toString());
});
p.stderr.on('data', (x) => {
process.stderr.write(x.toString());
});
p.on('exit', (code) => {
resolveFunc(code);
});
});
};
export const getFeedItemsMatchingRules = async (
feedItems: Array<FeedItem>,
rules: Array<Rule>,
): Array<PendingDownloadItems> => {
return feedItems.reduce((matchedItems: Array<PendingDownloadItems>, feedItem) => {
rules.forEach((rule) => {
const matchField = rule.field ? (feedItem[rule.field] as string) : (feedItem.title as string);
const isMatched = new RegExp(rule.match, 'gi').test(matchField);
const isExcluded = rule.exclude !== '' && new RegExp(rule.exclude, 'gi').test(matchField);
): Promise<Array<PendingDownloadItems>> => {
const matchedItems: Array<PendingDownloadItems> = [];
if (isMatched && !isExcluded) {
const torrentUrls = getTorrentUrlsFromFeedItem(feedItem);
const isAlreadyDownloaded = matchedItems.some((matchedItem) =>
torrentUrls.every((url) => matchedItem.urls.includes(url)),
);
await Promise.all(
feedItems.map(async (feedItem) => {
await Promise.all(
rules.map(async (rule) => {
const matchField = rule.field ? (feedItem[rule.field] as string) : (feedItem.title as string);
const isMatched = rule.match === '' || new RegExp(rule.match, 'gi').test(matchField);
const isExcluded = rule.exclude !== '' && new RegExp(rule.exclude, 'gi').test(matchField);
const scriptMatch = rule.script === '' || (await execAsync(rule.script, matchField)) === 80;
if (!isAlreadyDownloaded && torrentUrls[0] != null) {
matchedItems.push({
urls: torrentUrls as [string, ...string[]],
tags: rule.tags,
matchTitle: feedItem.title as string,
ruleID: rule._id,
ruleLabel: rule.label,
destination: rule.destination,
start: rule.startOnLoad,
});
}
}
});
if (isMatched && !isExcluded && scriptMatch) {
const torrentUrls = getTorrentUrlsFromFeedItem(feedItem);
const isAlreadyDownloaded = matchedItems.some((matchedItem) =>
torrentUrls.every((url) => matchedItem.urls.includes(url)),
);
return matchedItems;
}, []);
if (!isAlreadyDownloaded && torrentUrls[0] != null) {
matchedItems.push({
urls: torrentUrls as [string, ...string[]],
tags: rule.tags,
matchTitle: feedItem.title as string,
ruleID: rule._id,
ruleLabel: rule.label,
destination: rule.destination,
start: rule.startOnLoad,
});
}
}
}),
);
}),
);
return matchedItems;
};

2
shared/types/Feed.ts
View File

@@ -27,6 +27,8 @@ export interface Rule {
match: string;
// Regular expression to exclude items.
exclude: string;
// Custom script to select if the item should be downloaded (exit with status 80 to download).
script: string;
// Destination path where matched items are downloaded to.
destination: string;
// Tags to be added when items are queued for download.