amechama/sharkey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: allow setting separate timeout / max size for imports - fixes #479 (!519)

Browse Source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/519

Closes #479

Approved-by: Marie <marie@kaifa.ch>
Approved-by: Amelia Yukii <amelia.yukii@shourai.de>
This commit is contained in:
Amelia Yukii 2024-06-03 16:29:19 +00:00
commit 23b1c29a0b
4 changed files with 38 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.config/example.yml
View File

@ -287,5 +287,10 @@ checkActivityPubGetSignature: false
# Upload or download file size limits (bytes)
#maxFileSize: 262144000
# timeout and maximum size for imports (e.g. note imports)
#import:
# downloadTimeout: 30
# maxFileSize: 262144000
# PID File of master process
#pidFile: /tmp/misskey.pid

14
packages/backend/src/config.ts
View File

@ -97,6 +97,12 @@ type Source = {
perChannelMaxNoteCacheCount?: number;
perUserNotificationsMaxCount?: number;
deactivateAntennaThreshold?: number;
import?: {
downloadTimeout: number;
maxFileSize: number;
};
pidFile: string;
};
@ -177,6 +183,12 @@ export type Config = {
perChannelMaxNoteCacheCount: number;
perUserNotificationsMaxCount: number;
deactivateAntennaThreshold: number;
import: {
downloadTimeout: number;
maxFileSize: number;
} | undefined;
pidFile: string;
};
@ -284,6 +296,7 @@ export function loadConfig(): Config {
perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500,
deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7),
import: config.import,
pidFile: config.pidFile,
};
}
@ -427,4 +440,5 @@ function applyEnvOverrides(config: Source) {
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
}

8
packages/backend/src/core/DownloadService.ts
View File

@ -35,14 +35,14 @@ export class DownloadService {
}
@bindThis
public async downloadUrl(url: string, path: string): Promise<{
public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{
filename: string;
}> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = 30 * 1000;
const operationTimeout = 60 * 1000;
const maxSize = this.config.maxFileSize ?? 262144000;
const timeout = options.timeout ?? 30 * 1000;
const operationTimeout = options.operationTimeout ?? 60 * 1000;
const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000;
const urlObj = new URL(url);
let filename = urlObj.pathname.split('/').pop() ?? 'untitled';

21
packages/backend/src/queue/processors/ImportNotesProcessorService.ts
View File

@ -19,12 +19,16 @@ import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
import type { Config } from '@/config.js';
@Injectable()
export class ImportNotesProcessorService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@ -73,6 +77,11 @@ export class ImportNotesProcessorService {
}
}
@bindThis
private downloadUrl(url: string, path:string): Promise<{filename: string}> {
return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize });
}
@bindThis
private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> {
type NotesMap = {
@ -176,7 +185,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -206,7 +215,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -239,7 +248,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -297,7 +306,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(path, '', 'utf-8');
await this.downloadService.downloadUrl(file.url, path);
await this.downloadUrl(file.url, path);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -349,7 +358,7 @@ export class ImportNotesProcessorService {
if (!exists) {
try {
await this.downloadService.downloadUrl(file.url, filePath);
await this.downloadUrl(file.url, filePath);
} catch (e) { // TODO: 何度か再試行
this.logger.error(e instanceof Error ? e : new Error(e as string));
}
@ -488,7 +497,7 @@ export class ImportNotesProcessorService {
if (!exists) {
try {
await this.downloadService.downloadUrl(file.url, filePath);
await this.downloadUrl(file.url, filePath);
} catch (e) { // TODO: 何度か再試行
this.logger.error(e instanceof Error ? e : new Error(e as string));
}