server: rTorrent: save torrents and then add them with paths

Bug: #164, #741, #773

server/.jest/auth.setup.js

@@ -9,4 +9,8 @@ process.argv = ['node', 'flood'];
 process.argv.push('--rundir', temporaryRuntimeDirectory);
 process.argv.push('--noauth', 'false');
 
-afterAll(() => fs.rmdirSync(temporaryRuntimeDirectory, {recursive: true}));
+afterAll(() => {
+  // TODO: This leads test flakiness caused by ENOENT error
+  // NeDB provides no method to close database connection
+  fs.rmdirSync(temporaryRuntimeDirectory, {recursive: true});
+});

server/.jest/test.setup.js

@@ -37,6 +37,8 @@ process.argv.push('--rtsocket', rTorrentSocket);
 
 afterAll((done) => {
   rTorrentProcess.on('close', () => {
+    // TODO: This leads test flakiness caused by ENOENT error
+    // NeDB provides no method to close database connection
     fs.rmdirSync(temporaryRuntimeDirectory, {recursive: true});
     done();
   });

server/routes/api/torrents.test.ts

@@ -1,5 +1,6 @@
 import crypto from 'crypto';
 import fs from 'fs';
+import path from 'path';
 import readline from 'readline';
 import stream from 'stream';
 import supertest from 'supertest';
@@ -7,8 +8,13 @@ import supertest from 'supertest';
 import app from '../../app';
 import {getAuthToken} from './auth';
 import {getTempPath} from '../../models/TemporaryStorage';
+import paths from '../../../shared/config/paths';
 
-import type {AddTorrentByURLOptions, SetTorrentsTrackersOptions} from '../../../shared/types/api/torrents';
+import type {
+  AddTorrentByFileOptions,
+  AddTorrentByURLOptions,
+  SetTorrentsTrackersOptions,
+} from '../../../shared/types/api/torrents';
 import type {TorrentContent} from '../../../shared/types/TorrentContent';
 import type {TorrentList, TorrentProperties} from '../../../shared/types/Torrent';
 import type {TorrentStatus} from '../../../shared/constants/torrentStatusMap';
@@ -24,6 +30,13 @@ fs.mkdirSync(tempDirectory, {recursive: true});
 
 jest.setTimeout(20000);
 
+const torrentFiles = [
+  path.join(paths.appSrc, 'fixtures/single.torrent'),
+  path.join(paths.appSrc, 'fixtures/multi.torrent'),
+].map((torrentPath) => Buffer.from(fs.readFileSync(torrentPath)).toString('base64'));
+
+const torrentURLs = ['https://releases.ubuntu.com/20.04/ubuntu-20.04.1-live-server-amd64.iso.torrent'];
+
 let torrentHash = '';
 
 const activityStream = new stream.PassThrough();
@@ -32,7 +45,7 @@ request.get('/api/activity-stream').send().set('Cookie', [authToken]).pipe(activ
 
 describe('POST /api/torrents/add-urls', () => {
   const addTorrentByURLOptions: AddTorrentByURLOptions = {
-    urls: ['https://releases.ubuntu.com/20.04/ubuntu-20.04.1-live-server-amd64.iso.torrent'],
+    urls: torrentURLs,
     destination: tempDirectory,
     tags: ['test'],
     isBasePath: false,
@@ -41,7 +54,7 @@ describe('POST /api/torrents/add-urls', () => {
 
   const torrentAdded = new Promise((resolve) => {
     rl.on('line', (input) => {
-      if (input.includes('TORRENT_LIST_DIFF_CHANGE')) {
+      if (input.includes('TORRENT_LIST_ACTION_TORRENT_ADDED')) {
         resolve();
       }
     });
@@ -62,7 +75,7 @@ describe('POST /api/torrents/add-urls', () => {
       });
   });
 
-  it('GET /api/torrents', (done) => {
+  it('GET /api/torrents to verify torrents are added via URLs', (done) => {
     torrentAdded.then(() => {
       request
         .get('/api/torrents')
@@ -95,6 +108,63 @@ describe('POST /api/torrents/add-urls', () => {
   });
 });
 
+describe('POST /api/torrents/add-files', () => {
+  const addTorrentByFileOptions: AddTorrentByFileOptions = {
+    files: torrentFiles,
+    destination: tempDirectory,
+    tags: ['test'],
+    isBasePath: false,
+    start: false,
+  };
+
+  const torrentAdded = new Promise((resolve) => {
+    rl.on('line', (input) => {
+      if (input.includes('TORRENT_LIST_ACTION_TORRENT_ADDED')) {
+        resolve();
+      }
+    });
+  });
+
+  it('Adds a torrent from files', (done) => {
+    request
+      .post('/api/torrents/add-files')
+      .send(addTorrentByFileOptions)
+      .set('Cookie', [authToken])
+      .set('Accept', 'application/json')
+      .expect(200)
+      .expect('Content-Type', /json/)
+      .end((err, _res) => {
+        if (err) done(err);
+
+        done();
+      });
+  });
+
+  it('GET /api/torrents to verify torrents are added via files', (done) => {
+    torrentAdded.then(() => {
+      request
+        .get('/api/torrents')
+        .send()
+        .set('Cookie', [authToken])
+        .set('Accept', 'application/json')
+        .expect(200)
+        .expect('Content-Type', /json/)
+        .end((err, res) => {
+          if (err) done(err);
+
+          expect(res.body.torrents == null).toBe(false);
+          const torrentList: TorrentList = res.body.torrents;
+
+          expect(Object.keys(torrentList).length).toBeGreaterThanOrEqual(
+            torrentFiles.length + (torrentHash !== '' ? 1 : 0),
+          );
+
+          done();
+        });
+    });
+  });
+});
+
 describe('PATCH /api/torrents/trackers', () => {
   const testTrackers = [
     `https://${crypto.randomBytes(8).toString('hex')}.com/announce`,

server/services/rTorrent/clientGatewayService.ts

@@ -1,7 +1,8 @@
-import path from 'path';
+import crypto from 'crypto';
 import fs from 'fs';
 import geoip from 'geoip-country';
 import {moveSync} from 'fs-extra';
+import path from 'path';
 import sanitize from 'sanitize-filename';
 
 import type {ClientSettings} from '@shared/types/ClientSettings';
@@ -31,6 +32,7 @@ import ClientGatewayService from '../interfaces/clientGatewayService';
 import ClientRequestManager from './clientRequestManager';
 import scgiUtil from './util/scgiUtil';
 import {getMethodCalls, processMethodCallResponse} from './util/rTorrentMethodCallUtil';
+import {getTempPath} from '../../models/TemporaryStorage';
 import torrentFileUtil from '../../util/torrentFileUtil';
 import {
   encodeTags,
@@ -55,41 +57,25 @@ class RTorrentClientGatewayService extends ClientGatewayService {
   clientRequestManager = new ClientRequestManager(this.user.client as RTorrentConnectionSettings);
 
   async addTorrentsByFile({files, destination, tags, isBasePath, start}: AddTorrentByFileOptions): Promise<void> {
-    const destinationPath = sanitizePath(destination);
+    const tempPath = path.join(
+      getTempPath(this.user._id),
+      'torrents',
+      `${Date.now()}-${crypto.randomBytes(4).toString('hex')}`,
+    );
+    await createDirectory(tempPath);
 
-    if (!isAllowedPath(destinationPath)) {
-      throw accessDeniedError();
-    }
-
-    await createDirectory(destinationPath);
-
-    // Each torrent is sent individually because rTorrent might have small
-    // XMLRPC request size limit. This allows the user to send files reliably.
-    await Promise.all(
-      files.map(async (file) => {
-        const additionalCalls: Array<string> = [];
-
-        additionalCalls.push(`${isBasePath ? 'd.directory_base.set' : 'd.directory.set'}="${destinationPath}"`);
-
-        if (Array.isArray(tags)) {
-          additionalCalls.push(`d.custom1.set=${encodeTags(tags)}`);
-        }
-
-        additionalCalls.push(`d.custom.set=addtime,${Date.now() / 1000}`);
-
-        return (
-          this.clientRequestManager
-            .methodCall(
-              start ? 'load.raw_start' : 'load.raw',
-              ['', Buffer.from(file, 'base64')].concat(additionalCalls),
-            )
-            .then(this.processClientRequestSuccess, this.processClientRequestError)
-            .then(() => {
-              // returns nothing.
-            }) || Promise.reject()
-        );
+    const torrentPaths = await Promise.all(
+      files.map(async (file, index) => {
+        const torrentPath = path.join(tempPath, `${index}.torrent`);
+        fs.writeFileSync(torrentPath, Buffer.from(file, 'base64'), {});
+        return torrentPath;
       }),
     );
+
+    // Delete temp files after 5 minutes. This is more than enough.
+    setTimeout(() => fs.rmdirSync(tempPath, {recursive: true}), 1000 * 60 * 5);
+
+    return this.addTorrentsByURL({urls: torrentPaths, destination, tags, isBasePath, start});
   }
 
   async addTorrentsByURL({urls, destination, tags, isBasePath, start}: AddTorrentByURLOptions): Promise<void> {