amechama/sharkey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

separate character limits for local and remote notes

Browse Source
This commit is contained in:
Hazel K 2024-10-07 21:03:31 -04:00 committed by Hazelnoot
parent 55df1ad10f
commit 560ee43dcf
19 changed files with 142 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.config/ci.yml
View File

@ -167,8 +167,14 @@ id: 'aidx'
# IP address family used for outgoing request (ipv4, ipv6 or dual)
#outgoingAddressFamily: ipv4
# Amount of characters that can be used when writing notes (maximum: 100000, minimum: 1)
# Amount of characters that can be used when writing notes. Longer notes will be rejected. (minimum: 1)
maxNoteLength: 3000
# Amount of characters that will be saved for remote notes. Longer notes will be truncated to this length. (minimum: 1)
maxRemoteNoteLength: 100000
# Amount of characters that can be used when writing media descriptions (alt text). Longer descriptions will be rejected. (minimum: 1)
maxAltTextLength: 20000
# Amount of characters that will be saved for remote media descriptions (alt text). Longer descriptions will be truncated to this length. (minimum: 1)
maxRemoteAltTextLength: 100000
# Proxy for HTTP/HTTPS
#proxy: http://127.0.0.1:3128

8
.config/docker_example.yml
View File

@ -250,8 +250,14 @@ id: 'aidx'
# IP address family used for outgoing request (ipv4, ipv6 or dual)
#outgoingAddressFamily: ipv4
# Amount of characters that can be used when writing notes (maximum: 100000, minimum: 1)
# Amount of characters that can be used when writing notes. Longer notes will be rejected. (maximum: 100000, minimum: 1)
maxNoteLength: 3000
# Amount of characters that will be saved for remote notes. Longer notes will be truncated to this length. (maximum: 100000, minimum: 1)
maxRemoteNoteLength: 100000
# Amount of characters that can be used when writing media descriptions (alt text). Longer descriptions will be rejected. (minimum: 1)
maxAltTextLength: 20000
# Amount of characters that will be saved for remote media descriptions (alt text). Longer descriptions will be truncated to this length. (minimum: 1)
maxRemoteAltTextLength: 100000
# Proxy for HTTP/HTTPS
#proxy: http://127.0.0.1:3128

8
.config/example.yml
View File

@ -261,8 +261,14 @@ id: 'aidx'
# IP address family used for outgoing request (ipv4, ipv6 or dual)
#outgoingAddressFamily: ipv4
# Amount of characters that can be used when writing notes (maximum: 100000, minimum: 1)
# Amount of characters that can be used when writing notes. Longer notes will be rejected. (maximum: 100000, minimum: 1)
maxNoteLength: 3000
# Amount of characters that will be saved for remote notes. Longer notes will be truncated to this length. (maximum: 100000, minimum: 1)
maxRemoteNoteLength: 100000
# Amount of characters that can be used when writing media descriptions (alt text). Longer descriptions will be rejected. (minimum: 1)
maxAltTextLength: 20000
# Amount of characters that will be saved for remote media descriptions (alt text). Longer descriptions will be truncated to this length. (minimum: 1)
maxRemoteAltTextLength: 100000
# Proxy for HTTP/HTTPS
#proxy: http://127.0.0.1:3128

16
packages/backend/migration/1728348353115-soft-limit-drive-comment.js Normal file
View File

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class SoftLimitDriveComment1728348353115 {
name = 'SoftLimitDriveComment1728348353115'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "drive_file" ALTER COLUMN "comment" TYPE text`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "drive_file" ALTER COLUMN "comment" TYPE varchar(100000)`);
}
}

11
packages/backend/src/config.ts
View File

@ -73,6 +73,9 @@ type Source = {
maxFileSize?: number;
maxNoteLength?: number;
maxRemoteNoteLength?: number;
maxAltTextLength?: number;
maxRemoteAltTextLength?: number;
clusterLimit?: number;
@ -149,6 +152,9 @@ export type Config = {
allowedPrivateNetworks: string[] | undefined;
maxFileSize: number;
maxNoteLength: number;
maxRemoteNoteLength: number;
maxAltTextLength: number;
maxRemoteAltTextLength: number;
clusterLimit: number | undefined;
id: string;
outgoingAddress: string | undefined;
@ -301,6 +307,9 @@ export function loadConfig(): Config {
allowedPrivateNetworks: config.allowedPrivateNetworks,
maxFileSize: config.maxFileSize ?? 262144000,
maxNoteLength: config.maxNoteLength ?? 3000,
maxRemoteNoteLength: config.maxRemoteNoteLength ?? 100000,
maxAltTextLength: config.maxAltTextLength ?? 20000,
maxRemoteAltTextLength: config.maxRemoteAltTextLength ?? 100000,
clusterLimit: config.clusterLimit,
outgoingAddress: config.outgoingAddress,
outgoingAddressFamily: config.outgoingAddressFamily,
@ -475,7 +484,7 @@ function applyEnvOverrides(config: Source) {
_apply_top(['sentryForBackend', 'enableNodeProfiling']);
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'pidFile']]);
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature']]);
}

19
packages/backend/src/const.ts
View File

@ -3,30 +3,11 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
export const MAX_NOTE_TEXT_LENGTH = 3000;
export const USER_ONLINE_THRESHOLD = 1000 * 60 * 10; // 10min
export const USER_ACTIVE_THRESHOLD = 1000 * 60 * 60 * 24 * 3; // 3days
export const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16;
//#region hard limits
// If you change DB_* values, you must also change the DB schema.
/**
* Maximum note text length that can be stored in DB.
* Content Warnings are included in this limit.
* Surrogate pairs count as one
*/
export const DB_MAX_NOTE_TEXT_LENGTH = 100000;
/**
* Maximum image description length that can be stored in DB.
* Surrogate pairs count as one
*/
export const DB_MAX_IMAGE_COMMENT_LENGTH = 100000;
//#endregion
// ブラウザで直接表示することを許可するファイルの種類のリスト
// ここに含まれないものは application/octet-stream としてレスポンスされる
// SVGはXSSを生むので許可しない

13
packages/backend/src/core/NoteCreateService.ts
View File

@ -45,7 +45,6 @@ import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerServ
import { NoteReadService } from '@/core/NoteReadService.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { bindThis } from '@/decorators.js';
import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { RoleService } from '@/core/RoleService.js';
import { SearchService } from '@/core/SearchService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
@ -335,9 +334,13 @@ export class NoteCreateService implements OnApplicationShutdown {
data.localOnly = true;
}
const maxTextLength = user.host == null
? this.config.maxNoteLength
: this.config.maxRemoteNoteLength;
if (data.text) {
if (data.text.length > DB_MAX_NOTE_TEXT_LENGTH) {
data.text = data.text.slice(0, DB_MAX_NOTE_TEXT_LENGTH);
if (data.text.length > maxTextLength) {
data.text = data.text.slice(0, maxTextLength);
}
data.text = data.text.trim();
if (data.text === '') {
@ -348,8 +351,8 @@ export class NoteCreateService implements OnApplicationShutdown {
}
if (data.cw) {
if (data.cw.length > DB_MAX_NOTE_TEXT_LENGTH) {
data.cw = data.cw.slice(0, DB_MAX_NOTE_TEXT_LENGTH);
if (data.cw.length > maxTextLength) {
data.cw = data.cw.slice(0, maxTextLength);
}
data.cw = data.cw.trim();
if (data.cw === '') {

13
packages/backend/src/core/NoteEditService.ts
View File

@ -39,7 +39,6 @@ import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerServ
import { NoteReadService } from '@/core/NoteReadService.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { bindThis } from '@/decorators.js';
import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { RoleService } from '@/core/RoleService.js';
import { SearchService } from '@/core/SearchService.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
@ -365,9 +364,13 @@ export class NoteEditService implements OnApplicationShutdown {
data.localOnly = true;
}
const maxTextLength = user.host == null
? this.config.maxNoteLength
: this.config.maxRemoteNoteLength;
if (data.text) {
if (data.text.length > DB_MAX_NOTE_TEXT_LENGTH) {
data.text = data.text.slice(0, DB_MAX_NOTE_TEXT_LENGTH);
if (data.text.length > maxTextLength) {
data.text = data.text.slice(0, maxTextLength);
}
data.text = data.text.trim();
if (data.text === '') {
@ -378,8 +381,8 @@ export class NoteEditService implements OnApplicationShutdown {
}
if (data.cw) {
if (data.cw.length > DB_MAX_NOTE_TEXT_LENGTH) {
data.cw = data.cw.slice(0, DB_MAX_NOTE_TEXT_LENGTH);
if (data.cw.length > maxTextLength) {
data.cw = data.cw.slice(0, maxTextLength);
}
data.cw = data.cw.trim();
if (data.cw === '') {

6
packages/backend/src/core/activitypub/models/ApImageService.ts
View File

@ -9,12 +9,12 @@ import type { DriveFilesRepository, MiMeta } from '@/models/_.js';
import type { MiRemoteUser } from '@/models/User.js';
import type { MiDriveFile } from '@/models/DriveFile.js';
import { truncate } from '@/misc/truncate.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/const.js';
import { DriveService } from '@/core/DriveService.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { checkHttps } from '@/misc/check-https.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import type { Config } from '@/config.js';
import { ApResolverService } from '../ApResolverService.js';
import { ApLoggerService } from '../ApLoggerService.js';
import { isDocument, type IObject } from '../type.js';
@ -29,6 +29,8 @@ export class ApImageService {
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
@Inject(DI.config)
private config: Config,
private apResolverService: ApResolverService,
private driveService: DriveService,
@ -83,7 +85,7 @@ export class ApImageService {
uri: image.url,
sensitive: !!(image.sensitive),
isLink: !shouldBeCached,
comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH),
comment: truncate(image.name ?? undefined, this.config.maxRemoteAltTextLength),
});
if (!file.isLink || file.url === image.url) return file;

3
packages/backend/src/core/entities/MetaEntityService.ts
View File

@ -110,6 +110,9 @@ export class MetaEntityService {
backgroundImageUrl: instance.backgroundImageUrl,
logoImageUrl: instance.logoImageUrl,
maxNoteTextLength: this.config.maxNoteLength,
maxRemoteNoteTextLength: this.config.maxRemoteNoteLength,
maxAltTextLength: this.config.maxAltTextLength,
maxRemoteAltTextLength: this.config.maxRemoteAltTextLength,
defaultLightTheme,
defaultDarkTheme,
defaultLike: instance.defaultLike,

4
packages/backend/src/models/DriveFile.ts
View File

@ -4,7 +4,6 @@
*/
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/const.js';
import { id } from './util/id.js';
import { MiUser } from './User.js';
import { MiDriveFolder } from './DriveFolder.js';
@ -61,8 +60,7 @@ export class MiDriveFile {
})
public size: number;
@Column('varchar', {
length: DB_MAX_IMAGE_COMMENT_LENGTH,
@Column('text', {
nullable: true,
comment: 'The comment of the DriveFile.',
})

12
packages/backend/src/models/json-schema/meta.ts
View File

@ -168,6 +168,18 @@ export const packedMetaLiteSchema = {
type: 'number',
optional: false, nullable: false,
},
maxRemoteNoteTextLength: {
type: 'number',
optional: false, nullable: false,
},
maxAltTextLength: {
type: 'number',
optional: false, nullable: false,
},
maxRemoteAltTextLength: {
type: 'number',
optional: false, nullable: false,
},
ads: {
type: 'array',
optional: false, nullable: false,

3
packages/backend/src/server/NodeinfoServerService.ts
View File

@ -122,6 +122,9 @@ export class NodeinfoServerService {
enableMcaptcha: meta.enableMcaptcha,
enableTurnstile: meta.enableTurnstile,
maxNoteTextLength: this.config.maxNoteLength,
maxRemoteNoteTextLength: this.config.maxRemoteNoteLength,
maxAltTextLength: this.config.maxAltTextLength,
maxRemoteAltTextLength: this.config.maxRemoteAltTextLength,
enableEmail: meta.enableEmail,
enableServiceWorker: meta.enableServiceWorker,
proxyAccountName: proxyAccount ? proxyAccount.username : null,

17
packages/backend/src/server/api/endpoints/drive/files/create.ts
View File

@ -5,11 +5,11 @@
import ms from 'ms';
import { Inject, Injectable } from '@nestjs/common';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/const.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import { DriveService } from '@/core/DriveService.js';
import type { Config } from '@/config.js';
import { ApiError } from '../../../error.js';
import { MiMeta } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
@ -56,6 +56,12 @@ export const meta = {
code: 'NO_FREE_SPACE',
id: 'd08dbc37-a6a9-463a-8c47-96c32ab5f064',
},
commentTooLong: {
message: 'Cannot upload the file because the comment exceeds the instance limit.',
code: 'COMMENT_TOO_LONG',
id: 'sj3hsm2l-s83j-4sk3-sk3j-sn3k2k4nsm3l',
},
},
} as const;
@ -64,7 +70,7 @@ export const paramDef = {
properties: {
folderId: { type: 'string', format: 'misskey:id', nullable: true, default: null },
name: { type: 'string', nullable: true, default: null },
comment: { type: 'string', nullable: true, maxLength: DB_MAX_IMAGE_COMMENT_LENGTH, default: null },
comment: { type: 'string', nullable: true, default: null },
isSensitive: { type: 'boolean', default: false },
force: { type: 'boolean', default: false },
},
@ -77,6 +83,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
@Inject(DI.meta)
private serverSettings: MiMeta,
@Inject(DI.config)
private config: Config,
private driveFileEntityService: DriveFileEntityService,
private driveService: DriveService,
) {
@ -94,6 +103,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
}
if (ps.comment && ps.comment.length > this.config.maxAltTextLength) {
throw new ApiError(meta.errors.commentTooLong);
}
try {
// Create file
const driveFile = await this.driveService.addFile({

16
packages/backend/src/server/api/endpoints/drive/files/update.ts
View File

@ -9,8 +9,8 @@ import { Endpoint } from '@/server/api/endpoint-base.js';
import { DI } from '@/di-symbols.js';
import { RoleService } from '@/core/RoleService.js';
import { DriveService } from '@/core/DriveService.js';
import type { Config } from '@/config.js';
import { ApiError } from '../../../error.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/const.js';
export const meta = {
tags: ['drive'],
@ -51,6 +51,12 @@ export const meta = {
code: 'RESTRICTED_BY_ROLE',
id: '7f59dccb-f465-75ab-5cf4-3ce44e3282f7',
},
commentTooLong: {
message: 'Cannot upload the file because the comment exceeds the instance limit.',
code: 'COMMENT_TOO_LONG',
id: 'sj3hsm2l-s83j-4sk3-sk3j-sn3k2k4nsm3l',
},
},
res: {
type: 'object',
@ -66,7 +72,7 @@ export const paramDef = {
folderId: { type: 'string', format: 'misskey:id', nullable: true },
name: { type: 'string' },
isSensitive: { type: 'boolean' },
comment: { type: 'string', nullable: true, maxLength: DB_MAX_IMAGE_COMMENT_LENGTH },
comment: { type: 'string', nullable: true },
},
required: ['fileId'],
} as const;
@ -76,6 +82,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
constructor(
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
@Inject(DI.config)
private config: Config,
private driveService: DriveService,
private roleService: RoleService,
@ -90,6 +98,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError(meta.errors.accessDenied);
}
if (ps.comment && ps.comment.length > this.config.maxAltTextLength) {
throw new ApiError(meta.errors.commentTooLong);
}
let packedFile;
try {

23
packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts
View File

@ -4,12 +4,14 @@
*/
import ms from 'ms';
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import { DriveService } from '@/core/DriveService.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/const.js';
import { ApiError } from '@/server/api/error.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
export const meta = {
tags: ['drive'],
@ -26,6 +28,14 @@ export const meta = {
prohibitMoved: true,
kind: 'write:drive',
errors: {
commentTooLong: {
message: 'Cannot upload the file because the comment exceeds the instance limit.',
code: 'COMMENT_TOO_LONG',
id: 'sj3hsm2l-s83j-4sk3-sk3j-sn3k2k4nsm3l',
},
},
} as const;
export const paramDef = {
@ -34,7 +44,7 @@ export const paramDef = {
url: { type: 'string' },
folderId: { type: 'string', format: 'misskey:id', nullable: true, default: null },
isSensitive: { type: 'boolean', default: false },
comment: { type: 'string', nullable: true, maxLength: DB_MAX_IMAGE_COMMENT_LENGTH, default: null },
comment: { type: 'string', nullable: true, default: null },
marker: { type: 'string', nullable: true, default: null },
force: { type: 'boolean', default: false },
},
@ -44,11 +54,18 @@ export const paramDef = {
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
constructor(
@Inject(DI.config)
private config: Config,
private driveFileEntityService: DriveFileEntityService,
private driveService: DriveService,
private globalEventService: GlobalEventService,
) {
super(meta, paramDef, async (ps, user, _1, _2, _3, ip, headers) => {
if (ps.comment && ps.comment.length > this.config.maxAltTextLength) {
throw new ApiError(meta.errors.commentTooLong);
}
this.driveService.uploadFromUrl({ url: ps.url, user, folderId: ps.folderId, sensitive: ps.isSensitive, force: ps.force, comment: ps.comment, requestIp: ip, requestHeaders: headers }).then(file => {
this.driveFileEntityService.pack(file, { self: true }).then(packedFile => {
this.globalEventService.publishMainStream(user.id, 'urlUploadFinished', {

2
packages/backend/src/server/api/endpoints/notes/create.ts
View File

@ -147,7 +147,7 @@ export const paramDef = {
visibleUserIds: { type: 'array', uniqueItems: true, items: {
type: 'string', format: 'misskey:id',
} },
cw: { type: 'string', nullable: true, minLength: 1, maxLength: 500 },
cw: { type: 'string', nullable: true, minLength: 1 },
localOnly: { type: 'boolean', default: false },
reactionAcceptance: { type: 'string', nullable: true, enum: [null, 'likeOnly', 'likeOnlyForRemote', 'nonSensitiveOnly', 'nonSensitiveOnlyForLocalLikeOnlyForRemote'], default: null },
noExtractMentions: { type: 'boolean', default: false },

4
packages/backend/test/e2e/note.ts
View File

@ -9,10 +9,12 @@ process.env.NODE_ENV = 'test';
import * as assert from 'assert';
import { MiNote } from '@/models/Note.js';
import { MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { api, castAsError, initTestDb, post, role, signup, uploadFile, uploadUrl } from '../utils.js';
import type * as misskey from 'misskey-js';
// TODO: these tests are probably wrong for depending on this, but that's a problem for later.
const MAX_NOTE_TEXT_LENGTH = 3000;
describe('Note', () => {
let Notes: Repository<MiNote>;

3
packages/misskey-js/src/autogen/types.ts
View File

@ -5153,6 +5153,9 @@ export type components = {
iconUrl: string | null;
sidebarLogoUrl: string | null;
maxNoteTextLength: number;
maxRemoteNoteTextLength: number;
maxAltTextLength: number;
maxRemoteAltTextLength: number;
ads: {
/**
* Format: id