server: remove unused history snapshots and make it in-mem only

This commit is contained in:
Jesse Chan
2021-10-31 23:21:20 -07:00
parent 0cbd390e50
commit af8de75e05
10 changed files with 60 additions and 204 deletions
+13 -94
View File
@@ -1,8 +1,7 @@
import type {UserInDatabase} from '@shared/schema/Auth';
import type {TransferData, TransferSnapshot} from '@shared/types/TransferData';
import Datastore from 'nedb-promises';
import path from 'path';
import {setInterval} from 'timers';
import config from '../../config';
@@ -10,49 +9,18 @@ interface HistoryEraOpts {
interval: number;
maxTime: number;
name: string;
nextEraUpdateInterval?: number;
nextEra?: HistoryEra;
}
const MAX_NEXT_ERA_UPDATE_INTERVAL = 1000 * 60 * 60 * 12; // 12 hours
const CUMULATIVE_DATA_BUFFER_DIFF = 500; // 500 milliseconds
class HistoryEra {
data = [];
ready: Promise<void>;
lastUpdate = 0;
startedAt = Date.now();
opts: HistoryEraOpts;
db: Datastore;
autoCleanupInterval?: NodeJS.Timeout;
nextEraUpdateInterval?: NodeJS.Timeout;
private lastUpdate = 0;
private opts: HistoryEraOpts;
private db: Datastore;
constructor(user: UserInDatabase, opts: HistoryEraOpts) {
constructor(opts: HistoryEraOpts) {
this.opts = opts;
this.db = Datastore.create({
autoload: true,
filename: path.join(config.dbPath, user._id, 'history', `${opts.name}.db`),
});
this.ready = this.prepareDatabase();
}
private async prepareDatabase(): Promise<void> {
let lastUpdate = 0;
await this.db.find<TransferSnapshot>({}).then(
(snapshots) => {
snapshots.forEach((snapshot) => {
if (snapshot.timestamp > lastUpdate) {
lastUpdate = snapshot.timestamp;
}
});
this.lastUpdate = lastUpdate;
},
() => undefined,
);
await this.removeOutdatedData();
this.db = Datastore.create();
let cleanupInterval = this.opts.maxTime;
@@ -60,49 +28,18 @@ class HistoryEra {
cleanupInterval = config.dbCleanInterval;
}
this.autoCleanupInterval = setInterval(this.removeOutdatedData, cleanupInterval);
setInterval(this.removeOutdatedData, cleanupInterval);
}
private removeOutdatedData = async (): Promise<void> => {
if (this.opts.maxTime > 0) {
const minTimestamp = Date.now() - this.opts.maxTime;
return this.db.remove({timestamp: {$lt: minTimestamp}}, {multi: true}).then(
() => undefined,
() => undefined,
);
}
};
private updateNextEra = async (): Promise<void> => {
if (this.opts.nextEraUpdateInterval == null) {
return;
}
const minTimestamp = Date.now() - this.opts.nextEraUpdateInterval;
return this.db.find<TransferSnapshot>({timestamp: {$gte: minTimestamp}}).then((snapshots) => {
if (this.opts.nextEra == null) {
return;
}
let downTotal = 0;
let upTotal = 0;
snapshots.forEach((snapshot) => {
downTotal += Number(snapshot.download);
upTotal += Number(snapshot.upload);
});
this.opts.nextEra.addData({
download: Number(Number(downTotal / snapshots.length).toFixed(1)),
upload: Number(Number(upTotal / snapshots.length).toFixed(1)),
});
});
private removeOutdatedData = (): Promise<void> => {
const minTimestamp = Date.now() - this.opts.maxTime;
return this.db.remove({timestamp: {$lt: minTimestamp}}, {multi: true}).then(
() => undefined,
() => undefined,
);
};
async addData(data: TransferData): Promise<void> {
await this.ready;
const currentTime = Date.now();
if (currentTime - this.lastUpdate >= this.opts.interval - CUMULATIVE_DATA_BUFFER_DIFF) {
@@ -137,8 +74,6 @@ class HistoryEra {
}
async getData(): Promise<TransferSnapshot[]> {
await this.ready;
const minTimestamp = Date.now() - this.opts.maxTime;
return this.db
@@ -146,22 +81,6 @@ class HistoryEra {
.sort({timestamp: 1})
.then((snapshots) => snapshots.slice(snapshots.length - config.maxHistoryStates));
}
async setNextEra(nextEra: HistoryEra): Promise<void> {
await this.ready;
this.opts.nextEra = nextEra;
let {nextEraUpdateInterval} = this.opts;
if (nextEraUpdateInterval && nextEraUpdateInterval > MAX_NEXT_ERA_UPDATE_INTERVAL) {
nextEraUpdateInterval = MAX_NEXT_ERA_UPDATE_INTERVAL;
}
if (nextEraUpdateInterval) {
this.nextEraUpdateInterval = setInterval(this.updateNextEra, nextEraUpdateInterval);
}
}
}
export default HistoryEra;