From b7522f69e76f5b48c0b6a8fadcf5377c455988d6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 4 Feb 2023 10:02:03 +0900 Subject: [PATCH 1/6] fix typo --- packages/backend/src/core/AntennaService.ts | 6 ++--- .../src/core/CreateNotificationService.ts | 6 ++--- .../backend/src/core/DeleteAccountService.ts | 4 ++-- .../backend/src/core/NoteCreateService.ts | 10 ++++---- .../backend/src/core/NoteDeleteService.ts | 4 ++-- packages/backend/src/core/NoteReadService.ts | 18 +++++++------- packages/backend/src/core/PollService.ts | 4 ++-- packages/backend/src/core/ReactionService.ts | 6 ++--- .../backend/src/core/UserBlockingService.ts | 12 +++++----- .../backend/src/core/UserFollowingService.ts | 24 +++++++++---------- packages/backend/src/core/UserListService.ts | 4 ++-- .../backend/src/core/UserMutingService.ts | 2 +- 12 files changed, 50 insertions(+), 50 deletions(-) diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 7db8c43ea5..9544cba5ac 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -52,7 +52,7 @@ export class AntennaService implements OnApplicationShutdown { private utilityService: UtilityService, private idService: IdService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private pushNotificationService: PushNotificationService, private noteEntityService: NoteEntityService, private antennaEntityService: AntennaEntityService, @@ -109,7 +109,7 @@ export class AntennaService implements OnApplicationShutdown { read: read, }); - this.globalEventServie.publishAntennaStream(antenna.id, 'note', note); + this.globalEventService.publishAntennaStream(antenna.id, 'note', note); if (!read) { const mutings = await this.mutingsRepository.find({ @@ -139,7 +139,7 @@ export class AntennaService implements OnApplicationShutdown { setTimeout(async () => { const unread = await this.antennaNotesRepository.findOneBy({ antennaId: antenna.id, read: false }); if (unread) { - this.globalEventServie.publishMainStream(antenna.userId, 'unreadAntenna', antenna); + this.globalEventService.publishMainStream(antenna.userId, 'unreadAntenna', antenna); this.pushNotificationService.pushNotification(antenna.userId, 'unreadAntennaNote', { antenna: { id: antenna.id, name: antenna.name }, note: await this.noteEntityService.pack(note), diff --git a/packages/backend/src/core/CreateNotificationService.ts b/packages/backend/src/core/CreateNotificationService.ts index f376b7b9cf..cd47844a75 100644 --- a/packages/backend/src/core/CreateNotificationService.ts +++ b/packages/backend/src/core/CreateNotificationService.ts @@ -26,7 +26,7 @@ export class CreateNotificationService { private notificationEntityService: NotificationEntityService, private idService: IdService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private pushNotificationService: PushNotificationService, ) { } @@ -60,7 +60,7 @@ export class CreateNotificationService { const packed = await this.notificationEntityService.pack(notification, {}); // Publish notification event - this.globalEventServie.publishMainStream(notifieeId, 'notification', packed); + this.globalEventService.publishMainStream(notifieeId, 'notification', packed); // 2秒経っても(今回作成した)通知が既読にならなかったら「未読の通知がありますよ」イベントを発行する setTimeout(async () => { @@ -77,7 +77,7 @@ export class CreateNotificationService { } //#endregion - this.globalEventServie.publishMainStream(notifieeId, 'unreadNotification', packed); + this.globalEventService.publishMainStream(notifieeId, 'unreadNotification', packed); this.pushNotificationService.pushNotification(notifieeId, 'notification', packed); if (type === 'follow') this.emailNotificationFollow(notifieeId, await this.usersRepository.findOneByOrFail({ id: data.notifierId! })); diff --git a/packages/backend/src/core/DeleteAccountService.ts b/packages/backend/src/core/DeleteAccountService.ts index 0ac12857c9..2acb5f2303 100644 --- a/packages/backend/src/core/DeleteAccountService.ts +++ b/packages/backend/src/core/DeleteAccountService.ts @@ -14,7 +14,7 @@ export class DeleteAccountService { private userSuspendService: UserSuspendService, private queueService: QueueService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, ) { } @@ -38,6 +38,6 @@ export class DeleteAccountService { }); // Terminate streaming - this.globalEventServie.publishUserEvent(user.id, 'terminate', {}); + this.globalEventService.publishUserEvent(user.id, 'terminate', {}); } } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 3dc44a25fe..4a81f764dc 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -175,7 +175,7 @@ export class NoteCreateService { private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private idService: IdService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private queueService: QueueService, private noteReadService: NoteReadService, private createNotificationService: CreateNotificationService, @@ -535,7 +535,7 @@ export class NoteCreateService { // Pack the note const noteObj = await this.noteEntityService.pack(note); - this.globalEventServie.publishNotesStream(noteObj); + this.globalEventService.publishNotesStream(noteObj); this.webhookService.getActiveWebhooks().then(webhooks => { webhooks = webhooks.filter(x => x.userId === user.id && x.on.includes('note')); @@ -561,7 +561,7 @@ export class NoteCreateService { if (!threadMuted) { nm.push(data.reply.userId, 'reply'); - this.globalEventServie.publishMainStream(data.reply.userId, 'reply', noteObj); + this.globalEventService.publishMainStream(data.reply.userId, 'reply', noteObj); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === data.reply!.userId && x.on.includes('reply')); for (const webhook of webhooks) { @@ -584,7 +584,7 @@ export class NoteCreateService { // Publish event if ((user.id !== data.renote.userId) && data.renote.userHost === null) { - this.globalEventServie.publishMainStream(data.renote.userId, 'renote', noteObj); + this.globalEventService.publishMainStream(data.renote.userId, 'renote', noteObj); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === data.renote!.userId && x.on.includes('renote')); for (const webhook of webhooks) { @@ -684,7 +684,7 @@ export class NoteCreateService { detail: true, }); - this.globalEventServie.publishMainStream(u.id, 'mention', detailPackedNote); + this.globalEventService.publishMainStream(u.id, 'mention', detailPackedNote); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === u.id && x.on.includes('mention')); for (const webhook of webhooks) { diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index b1f16b6e8a..4dad825097 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -34,7 +34,7 @@ export class NoteDeleteService { private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private relayService: RelayService, private federatedInstanceService: FederatedInstanceService, private apRendererService: ApRendererService, @@ -63,7 +63,7 @@ export class NoteDeleteService { } if (!quiet) { - this.globalEventServie.publishNoteStream(note.id, 'deleted', { + this.globalEventService.publishNoteStream(note.id, 'deleted', { deletedAt: deletedAt, }); diff --git a/packages/backend/src/core/NoteReadService.ts b/packages/backend/src/core/NoteReadService.ts index f4395725d6..84983d600e 100644 --- a/packages/backend/src/core/NoteReadService.ts +++ b/packages/backend/src/core/NoteReadService.ts @@ -40,7 +40,7 @@ export class NoteReadService { private userEntityService: UserEntityService, private idService: IdService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private notificationService: NotificationService, private antennaService: AntennaService, private pushNotificationService: PushNotificationService, @@ -87,13 +87,13 @@ export class NoteReadService { if (exist == null) return; if (params.isMentioned) { - this.globalEventServie.publishMainStream(userId, 'unreadMention', note.id); + this.globalEventService.publishMainStream(userId, 'unreadMention', note.id); } if (params.isSpecified) { - this.globalEventServie.publishMainStream(userId, 'unreadSpecifiedNote', note.id); + this.globalEventService.publishMainStream(userId, 'unreadSpecifiedNote', note.id); } if (note.channelId) { - this.globalEventServie.publishMainStream(userId, 'unreadChannel', note.id); + this.globalEventService.publishMainStream(userId, 'unreadChannel', note.id); } }, 2000); } @@ -155,7 +155,7 @@ export class NoteReadService { }).then(mentionsCount => { if (mentionsCount === 0) { // 全て既読になったイベントを発行 - this.globalEventServie.publishMainStream(userId, 'readAllUnreadMentions'); + this.globalEventService.publishMainStream(userId, 'readAllUnreadMentions'); } }); @@ -165,7 +165,7 @@ export class NoteReadService { }).then(specifiedCount => { if (specifiedCount === 0) { // 全て既読になったイベントを発行 - this.globalEventServie.publishMainStream(userId, 'readAllUnreadSpecifiedNotes'); + this.globalEventService.publishMainStream(userId, 'readAllUnreadSpecifiedNotes'); } }); @@ -175,7 +175,7 @@ export class NoteReadService { }).then(channelNoteCount => { if (channelNoteCount === 0) { // 全て既読になったイベントを発行 - this.globalEventServie.publishMainStream(userId, 'readAllChannels'); + this.globalEventService.publishMainStream(userId, 'readAllChannels'); } }); @@ -200,14 +200,14 @@ export class NoteReadService { }); if (count === 0) { - this.globalEventServie.publishMainStream(userId, 'readAntenna', antenna); + this.globalEventService.publishMainStream(userId, 'readAntenna', antenna); this.pushNotificationService.pushNotification(userId, 'readAntenna', { antennaId: antenna.id }); } } this.userEntityService.getHasUnreadAntenna(userId).then(unread => { if (!unread) { - this.globalEventServie.publishMainStream(userId, 'readAllAntennas'); + this.globalEventService.publishMainStream(userId, 'readAllAntennas'); this.pushNotificationService.pushNotification(userId, 'readAllAntennas', undefined); } }); diff --git a/packages/backend/src/core/PollService.ts b/packages/backend/src/core/PollService.ts index abc598ab76..2bd0675505 100644 --- a/packages/backend/src/core/PollService.ts +++ b/packages/backend/src/core/PollService.ts @@ -34,7 +34,7 @@ export class PollService { private userEntityService: UserEntityService, private idService: IdService, private relayService: RelayService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private createNotificationService: CreateNotificationService, private apRendererService: ApRendererService, private apDeliverManagerService: ApDeliverManagerService, @@ -88,7 +88,7 @@ export class PollService { const index = choice + 1; // In SQL, array index is 1 based await this.pollsRepository.query(`UPDATE poll SET votes[${index}] = votes[${index}] + 1 WHERE "noteId" = '${poll.noteId}'`); - this.globalEventServie.publishNoteStream(note.id, 'pollVoted', { + this.globalEventService.publishNoteStream(note.id, 'pollVoted', { choice: choice, userId: user.id, }); diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 0c1c3d0a3b..c280fc6832 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -74,7 +74,7 @@ export class ReactionService { private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private idService: IdService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private apRendererService: ApRendererService, private apDeliverManagerService: ApDeliverManagerService, private createNotificationService: CreateNotificationService, @@ -157,7 +157,7 @@ export class ReactionService { select: ['name', 'host', 'originalUrl', 'publicUrl'], }); - this.globalEventServie.publishNoteStream(note.id, 'reacted', { + this.globalEventService.publishNoteStream(note.id, 'reacted', { reaction: decodedReaction.reaction, emoji: emoji != null ? { name: emoji.host ? `${emoji.name}@${emoji.host}` : `${emoji.name}@.`, @@ -229,7 +229,7 @@ export class ReactionService { if (!user.isBot) this.notesRepository.decrement({ id: note.id }, 'score', 1); - this.globalEventServie.publishNoteStream(note.id, 'unreacted', { + this.globalEventService.publishNoteStream(note.id, 'unreacted', { reaction: this.decodeReaction(exist.reaction).reaction, userId: user.id, }); diff --git a/packages/backend/src/core/UserBlockingService.ts b/packages/backend/src/core/UserBlockingService.ts index c923700427..a65a0bf313 100644 --- a/packages/backend/src/core/UserBlockingService.ts +++ b/packages/backend/src/core/UserBlockingService.ts @@ -42,7 +42,7 @@ export class UserBlockingService { private userEntityService: UserEntityService, private idService: IdService, private queueService: QueueService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private webhookService: WebhookService, private apRendererService: ApRendererService, private perUserFollowingChart: PerUserFollowingChart, @@ -97,15 +97,15 @@ export class UserBlockingService { if (this.userEntityService.isLocalUser(followee)) { this.userEntityService.pack(followee, followee, { detail: true, - }).then(packed => this.globalEventServie.publishMainStream(followee.id, 'meUpdated', packed)); + }).then(packed => this.globalEventService.publishMainStream(followee.id, 'meUpdated', packed)); } if (this.userEntityService.isLocalUser(follower)) { this.userEntityService.pack(followee, follower, { detail: true, }).then(async packed => { - this.globalEventServie.publishUserEvent(follower.id, 'unfollow', packed); - this.globalEventServie.publishMainStream(follower.id, 'unfollow', packed); + this.globalEventService.publishUserEvent(follower.id, 'unfollow', packed); + this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); for (const webhook of webhooks) { @@ -152,8 +152,8 @@ export class UserBlockingService { this.userEntityService.pack(followee, follower, { detail: true, }).then(async packed => { - this.globalEventServie.publishUserEvent(follower.id, 'unfollow', packed); - this.globalEventServie.publishMainStream(follower.id, 'unfollow', packed); + this.globalEventService.publishUserEvent(follower.id, 'unfollow', packed); + this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); for (const webhook of webhooks) { diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index f1ce311cea..205592b231 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -57,7 +57,7 @@ export class UserFollowingService { private userEntityService: UserEntityService, private idService: IdService, private queueService: QueueService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private createNotificationService: CreateNotificationService, private federatedInstanceService: FederatedInstanceService, private webhookService: WebhookService, @@ -227,8 +227,8 @@ export class UserFollowingService { this.userEntityService.pack(followee.id, follower, { detail: true, }).then(async packed => { - this.globalEventServie.publishUserEvent(follower.id, 'follow', packed as Packed<'UserDetailedNotMe'>); - this.globalEventServie.publishMainStream(follower.id, 'follow', packed as Packed<'UserDetailedNotMe'>); + this.globalEventService.publishUserEvent(follower.id, 'follow', packed as Packed<'UserDetailedNotMe'>); + this.globalEventService.publishMainStream(follower.id, 'follow', packed as Packed<'UserDetailedNotMe'>); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('follow')); for (const webhook of webhooks) { @@ -242,7 +242,7 @@ export class UserFollowingService { // Publish followed event if (this.userEntityService.isLocalUser(followee)) { this.userEntityService.pack(follower.id, followee).then(async packed => { - this.globalEventServie.publishMainStream(followee.id, 'followed', packed); + this.globalEventService.publishMainStream(followee.id, 'followed', packed); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === followee.id && x.on.includes('followed')); for (const webhook of webhooks) { @@ -288,8 +288,8 @@ export class UserFollowingService { this.userEntityService.pack(followee.id, follower, { detail: true, }).then(async packed => { - this.globalEventServie.publishUserEvent(follower.id, 'unfollow', packed); - this.globalEventServie.publishMainStream(follower.id, 'unfollow', packed); + this.globalEventService.publishUserEvent(follower.id, 'unfollow', packed); + this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); for (const webhook of webhooks) { @@ -388,11 +388,11 @@ export class UserFollowingService { // Publish receiveRequest event if (this.userEntityService.isLocalUser(followee)) { - this.userEntityService.pack(follower.id, followee).then(packed => this.globalEventServie.publishMainStream(followee.id, 'receiveFollowRequest', packed)); + this.userEntityService.pack(follower.id, followee).then(packed => this.globalEventService.publishMainStream(followee.id, 'receiveFollowRequest', packed)); this.userEntityService.pack(followee.id, followee, { detail: true, - }).then(packed => this.globalEventServie.publishMainStream(followee.id, 'meUpdated', packed)); + }).then(packed => this.globalEventService.publishMainStream(followee.id, 'meUpdated', packed)); // 通知を作成 this.createNotificationService.createNotification(followee.id, 'receiveFollowRequest', { @@ -440,7 +440,7 @@ export class UserFollowingService { this.userEntityService.pack(followee.id, followee, { detail: true, - }).then(packed => this.globalEventServie.publishMainStream(followee.id, 'meUpdated', packed)); + }).then(packed => this.globalEventService.publishMainStream(followee.id, 'meUpdated', packed)); } @bindThis @@ -468,7 +468,7 @@ export class UserFollowingService { this.userEntityService.pack(followee.id, followee, { detail: true, - }).then(packed => this.globalEventServie.publishMainStream(followee.id, 'meUpdated', packed)); + }).then(packed => this.globalEventService.publishMainStream(followee.id, 'meUpdated', packed)); } @bindThis @@ -583,8 +583,8 @@ export class UserFollowingService { detail: true, }); - this.globalEventServie.publishUserEvent(follower.id, 'unfollow', packedFollowee); - this.globalEventServie.publishMainStream(follower.id, 'unfollow', packedFollowee); + this.globalEventService.publishUserEvent(follower.id, 'unfollow', packedFollowee); + this.globalEventService.publishMainStream(follower.id, 'unfollow', packedFollowee); const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); for (const webhook of webhooks) { diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index fc48738307..c174394999 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -25,7 +25,7 @@ export class UserListService { private idService: IdService, private userFollowingService: UserFollowingService, private roleService: RoleService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, private proxyAccountService: ProxyAccountService, ) { } @@ -46,7 +46,7 @@ export class UserListService { userListId: list.id, } as UserListJoining); - this.globalEventServie.publishUserListStream(list.id, 'userAdded', await this.userEntityService.pack(target)); + this.globalEventService.publishUserListStream(list.id, 'userAdded', await this.userEntityService.pack(target)); // このインスタンス内にこのリモートユーザーをフォローしているユーザーがいなくても投稿を受け取るためにダミーのユーザーがフォローしたということにする if (this.userEntityService.isRemoteUser(target)) { diff --git a/packages/backend/src/core/UserMutingService.ts b/packages/backend/src/core/UserMutingService.ts index 3029d02c00..e98f11709f 100644 --- a/packages/backend/src/core/UserMutingService.ts +++ b/packages/backend/src/core/UserMutingService.ts @@ -18,7 +18,7 @@ export class UserMutingService { private idService: IdService, private queueService: QueueService, - private globalEventServie: GlobalEventService, + private globalEventService: GlobalEventService, ) { } From 0c12e8010606ecb54b9d0167786d446e660de9c8 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 4 Feb 2023 12:40:40 +0900 Subject: [PATCH 2/6] perf(server): cache blocking --- packages/backend/src/core/AntennaService.ts | 12 +--- packages/backend/src/core/PollService.ts | 16 ++--- packages/backend/src/core/ReactionService.ts | 11 ++- .../backend/src/core/UserBlockingService.ts | 71 ++++++++++++++++++- .../backend/src/core/UserFollowingService.ts | 36 +++------- packages/backend/src/misc/cache.ts | 2 + .../server/api/endpoints/notes/polls/vote.ts | 14 ++-- .../backend/src/server/api/stream/types.ts | 2 + 8 files changed, 99 insertions(+), 65 deletions(-) diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 9544cba5ac..a71327e947 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -10,10 +10,9 @@ import { isUserRelated } from '@/misc/is-user-related.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { PushNotificationService } from '@/core/PushNotificationService.js'; import * as Acct from '@/misc/acct.js'; -import { Cache } from '@/misc/cache.js'; import type { Packed } from '@/misc/schema.js'; import { DI } from '@/di-symbols.js'; -import type { MutingsRepository, BlockingsRepository, NotesRepository, AntennaNotesRepository, AntennasRepository, UserGroupJoiningsRepository, UserListJoiningsRepository } from '@/models/index.js'; +import type { MutingsRepository, NotesRepository, AntennaNotesRepository, AntennasRepository, UserGroupJoiningsRepository, UserListJoiningsRepository } from '@/models/index.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { StreamMessages } from '@/server/api/stream/types.js'; @@ -23,7 +22,6 @@ import type { OnApplicationShutdown } from '@nestjs/common'; export class AntennaService implements OnApplicationShutdown { private antennasFetched: boolean; private antennas: Antenna[]; - private blockingCache: Cache; constructor( @Inject(DI.redisSubscriber) @@ -32,9 +30,6 @@ export class AntennaService implements OnApplicationShutdown { @Inject(DI.mutingsRepository) private mutingsRepository: MutingsRepository, - @Inject(DI.blockingsRepository) - private blockingsRepository: BlockingsRepository, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -59,7 +54,6 @@ export class AntennaService implements OnApplicationShutdown { ) { this.antennasFetched = false; this.antennas = []; - this.blockingCache = new Cache(1000 * 60 * 5); this.redisSubscriber.on('message', this.onRedisMessage); } @@ -155,10 +149,6 @@ export class AntennaService implements OnApplicationShutdown { public async checkHitAntenna(antenna: Antenna, note: (Note | Packed<'Note'>), noteUser: { id: User['id']; username: string; host: string | null; }): Promise { if (note.visibility === 'specified') return false; if (note.visibility === 'followers') return false; - - // アンテナ作成者がノート作成者にブロックされていたらスキップ - const blockings = await this.blockingCache.fetch(noteUser.id, () => this.blockingsRepository.findBy({ blockerId: noteUser.id }).then(res => res.map(x => x.blockeeId))); - if (blockings.some(blocking => blocking === antenna.userId)) return false; if (!antenna.withReplies && note.replyId != null) return false; diff --git a/packages/backend/src/core/PollService.ts b/packages/backend/src/core/PollService.ts index 2bd0675505..042dcb3e67 100644 --- a/packages/backend/src/core/PollService.ts +++ b/packages/backend/src/core/PollService.ts @@ -1,17 +1,17 @@ import { Inject, Injectable } from '@nestjs/common'; import { Not } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { NotesRepository, UsersRepository, BlockingsRepository, PollsRepository, PollVotesRepository } from '@/models/index.js'; +import type { NotesRepository, UsersRepository, PollsRepository, PollVotesRepository } from '@/models/index.js'; import type { Note } from '@/models/entities/Note.js'; import { RelayService } from '@/core/RelayService.js'; import type { CacheableUser } from '@/models/entities/User.js'; import { IdService } from '@/core/IdService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; -import { CreateNotificationService } from '@/core/CreateNotificationService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { bindThis } from '@/decorators.js'; +import { UserBlockingService } from '@/core/UserBlockingService.js'; @Injectable() export class PollService { @@ -28,14 +28,11 @@ export class PollService { @Inject(DI.pollVotesRepository) private pollVotesRepository: PollVotesRepository, - @Inject(DI.blockingsRepository) - private blockingsRepository: BlockingsRepository, - private userEntityService: UserEntityService, private idService: IdService, private relayService: RelayService, private globalEventService: GlobalEventService, - private createNotificationService: CreateNotificationService, + private userBlockingService: UserBlockingService, private apRendererService: ApRendererService, private apDeliverManagerService: ApDeliverManagerService, ) { @@ -52,11 +49,8 @@ export class PollService { // Check blocking if (note.userId !== user.id) { - const block = await this.blockingsRepository.findOneBy({ - blockerId: note.userId, - blockeeId: user.id, - }); - if (block) { + const blocked = await this.userBlockingService.checkBlocked(note.userId, user.id); + if (blocked) { throw new Error('blocked'); } } diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index c280fc6832..3806590059 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -18,7 +18,8 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { MetaService } from '@/core/MetaService.js'; import { bindThis } from '@/decorators.js'; -import { UtilityService } from './UtilityService.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { UserBlockingService } from '@/core/UserBlockingService.js'; const legacies: Record = { 'like': '👍', @@ -73,6 +74,7 @@ export class ReactionService { private metaService: MetaService, private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, + private userBlockingService: UserBlockingService, private idService: IdService, private globalEventService: GlobalEventService, private apRendererService: ApRendererService, @@ -86,11 +88,8 @@ export class ReactionService { public async create(user: { id: User['id']; host: User['host']; isBot: User['isBot'] }, note: Note, reaction?: string) { // Check blocking if (note.userId !== user.id) { - const block = await this.blockingsRepository.findOneBy({ - blockerId: note.userId, - blockeeId: user.id, - }); - if (block) { + const blocked = await this.userBlockingService.checkBlocked(note.userId, user.id); + if (blocked) { throw new IdentifiableError('e70412a4-7197-4726-8e74-f3e0deb92aa7'); } } diff --git a/packages/backend/src/core/UserBlockingService.ts b/packages/backend/src/core/UserBlockingService.ts index a65a0bf313..d734328669 100644 --- a/packages/backend/src/core/UserBlockingService.ts +++ b/packages/backend/src/core/UserBlockingService.ts @@ -1,5 +1,6 @@ -import { Inject, Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import Redis from 'ioredis'; import { IdService } from '@/core/IdService.js'; import type { CacheableUser, User } from '@/models/entities/User.js'; import type { Blocking } from '@/models/entities/Blocking.js'; @@ -7,7 +8,6 @@ import { QueueService } from '@/core/QueueService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import PerUserFollowingChart from '@/core/chart/charts/per-user-following.js'; import { DI } from '@/di-symbols.js'; -import logger from '@/logger.js'; import type { UsersRepository, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, UserListsRepository, UserListJoiningsRepository } from '@/models/index.js'; import Logger from '@/logger.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; @@ -15,12 +15,20 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { WebhookService } from '@/core/WebhookService.js'; import { bindThis } from '@/decorators.js'; +import { Cache } from '@/misc/cache.js'; +import { StreamMessages } from '@/server/api/stream/types.js'; @Injectable() -export class UserBlockingService { +export class UserBlockingService implements OnApplicationShutdown { private logger: Logger; + // キーがユーザーIDで、値がそのユーザーがブロックしているユーザーのIDのリストなキャッシュ + private blockingsByUserIdCache: Cache; + constructor( + @Inject(DI.redisSubscriber) + private redisSubscriber: Redis.Redis, + @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -49,6 +57,37 @@ export class UserBlockingService { private loggerService: LoggerService, ) { this.logger = this.loggerService.getLogger('user-block'); + + this.blockingsByUserIdCache = new Cache(Infinity); + + this.redisSubscriber.on('message', this.onMessage); + } + + @bindThis + private async onMessage(_: string, data: string): Promise { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as StreamMessages['internal']['payload']; + switch (type) { + case 'blockingCreated': { + const cached = this.blockingsByUserIdCache.get(body.blockerId); + if (cached) { + this.blockingsByUserIdCache.set(body.blockerId, [...cached, ...[body.blockeeId]]); + } + break; + } + case 'blockingDeleted': { + const cached = this.blockingsByUserIdCache.get(body.blockerId); + if (cached) { + this.blockingsByUserIdCache.set(body.blockerId, cached.filter(x => x !== body.blockeeId)); + } + break; + } + default: + break; + } + } } @bindThis @@ -72,6 +111,11 @@ export class UserBlockingService { await this.blockingsRepository.insert(blocking); + this.globalEventService.publishInternalEvent('blockingCreated', { + blockerId: blocker.id, + blockeeId: blockee.id, + }); + if (this.userEntityService.isLocalUser(blocker) && this.userEntityService.isRemoteUser(blockee)) { const content = this.apRendererService.renderActivity(this.apRendererService.renderBlock(blocking)); this.queueService.deliver(blocker, content, blockee.inbox); @@ -210,10 +254,31 @@ export class UserBlockingService { await this.blockingsRepository.delete(blocking.id); + this.globalEventService.publishInternalEvent('blockingDeleted', { + blockerId: blocker.id, + blockeeId: blockee.id, + }); + // deliver if remote bloking if (this.userEntityService.isLocalUser(blocker) && this.userEntityService.isRemoteUser(blockee)) { const content = this.apRendererService.renderActivity(this.apRendererService.renderUndo(this.apRendererService.renderBlock(blocking), blocker)); this.queueService.deliver(blocker, content, blockee.inbox); } } + + @bindThis + public async checkBlocked(blockerId: User['id'], blockeeId: User['id']): Promise { + const blockedUserIds = await this.blockingsByUserIdCache.fetch(blockerId, () => this.blockingsRepository.find({ + where: { + blockerId, + }, + select: ['blockeeId'], + }).then(records => records.map(record => record.blockeeId))); + return blockedUserIds.includes(blockeeId); + } + + @bindThis + public onApplicationShutdown(signal?: string | undefined) { + this.redisSubscriber.off('message', this.onMessage); + } } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 205592b231..18a2ef5c05 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -12,10 +12,11 @@ import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { WebhookService } from '@/core/WebhookService.js'; import { CreateNotificationService } from '@/core/CreateNotificationService.js'; import { DI } from '@/di-symbols.js'; -import type { BlockingsRepository, FollowingsRepository, FollowRequestsRepository, InstancesRepository, UserProfilesRepository, UsersRepository } from '@/models/index.js'; +import type { FollowingsRepository, FollowRequestsRepository, InstancesRepository, UserProfilesRepository, UsersRepository } from '@/models/index.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { bindThis } from '@/decorators.js'; +import { UserBlockingService } from '@/core/UserBlockingService.js'; import Logger from '../logger.js'; const logger = new Logger('following/create'); @@ -48,13 +49,11 @@ export class UserFollowingService { @Inject(DI.followRequestsRepository) private followRequestsRepository: FollowRequestsRepository, - @Inject(DI.blockingsRepository) - private blockingsRepository: BlockingsRepository, - @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, private userEntityService: UserEntityService, + private userBlockingService: UserBlockingService, private idService: IdService, private queueService: QueueService, private globalEventService: GlobalEventService, @@ -62,7 +61,6 @@ export class UserFollowingService { private federatedInstanceService: FederatedInstanceService, private webhookService: WebhookService, private apRendererService: ApRendererService, - private globalEventService: GlobalEventService, private perUserFollowingChart: PerUserFollowingChart, private instanceChart: InstanceChart, ) { @@ -77,26 +75,20 @@ export class UserFollowingService { // check blocking const [blocking, blocked] = await Promise.all([ - this.blockingsRepository.findOneBy({ - blockerId: follower.id, - blockeeId: followee.id, - }), - this.blockingsRepository.findOneBy({ - blockerId: followee.id, - blockeeId: follower.id, - }), + this.userBlockingService.checkBlocked(follower.id, followee.id), + this.userBlockingService.checkBlocked(followee.id, follower.id), ]); if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee) && blocked) { - // リモートフォローを受けてブロックしていた場合は、エラーにするのではなくRejectを送り返しておしまい。 + // リモートフォローを受けてブロックしていた場合は、エラーにするのではなくRejectを送り返しておしまい。 const content = this.apRendererService.renderActivity(this.apRendererService.renderReject(this.apRendererService.renderFollow(follower, followee, requestId), followee)); this.queueService.deliver(followee, content, follower.inbox); return; } else if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee) && blocking) { - // リモートフォローを受けてブロックされているはずの場合だったら、ブロック解除しておく。 - await this.blockingsRepository.delete(blocking.id); + // リモートフォローを受けてブロックされているはずの場合だったら、ブロック解除しておく。 + await this.userBlockingService.unblock(follower, followee); } else { - // それ以外は単純に例外 + // それ以外は単純に例外 if (blocking != null) throw new IdentifiableError('710e8fb0-b8c3-4922-be49-d5d93d8e6a6e', 'blocking'); if (blocked != null) throw new IdentifiableError('3338392a-f764-498d-8855-db939dcf8c48', 'blocked'); } @@ -357,14 +349,8 @@ export class UserFollowingService { // check blocking const [blocking, blocked] = await Promise.all([ - this.blockingsRepository.findOneBy({ - blockerId: follower.id, - blockeeId: followee.id, - }), - this.blockingsRepository.findOneBy({ - blockerId: followee.id, - blockeeId: follower.id, - }), + this.userBlockingService.checkBlocked(follower.id, followee.id), + this.userBlockingService.checkBlocked(followee.id, follower.id), ]); if (blocking != null) throw new Error('blocking'); diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 69512498f8..43a71a2b57 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -1,5 +1,7 @@ import { bindThis } from '@/decorators.js'; +// TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？ + export class Cache { public cache: Map; private lifetime: number; diff --git a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts index d583dfb936..befaea4664 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts @@ -1,6 +1,6 @@ import { Not } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import type { UsersRepository, BlockingsRepository, PollsRepository, PollVotesRepository } from '@/models/index.js'; +import type { UsersRepository, PollsRepository, PollVotesRepository } from '@/models/index.js'; import type { IRemoteUser } from '@/models/entities/User.js'; import { IdService } from '@/core/IdService.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; @@ -11,6 +11,7 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { CreateNotificationService } from '@/core/CreateNotificationService.js'; import { DI } from '@/di-symbols.js'; +import { UserBlockingService } from '@/core/UserBlockingService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -77,9 +78,6 @@ export default class extends Endpoint { @Inject(DI.usersRepository) private usersRepository: UsersRepository, - @Inject(DI.blockingsRepository) - private blockingsRepository: BlockingsRepository, - @Inject(DI.pollsRepository) private pollsRepository: PollsRepository, @@ -93,6 +91,7 @@ export default class extends Endpoint { private apRendererService: ApRendererService, private globalEventService: GlobalEventService, private createNotificationService: CreateNotificationService, + private userBlockingService: UserBlockingService, ) { super(meta, paramDef, async (ps, me) => { const createdAt = new Date(); @@ -109,11 +108,8 @@ export default class extends Endpoint { // Check blocking if (note.userId !== me.id) { - const block = await this.blockingsRepository.findOneBy({ - blockerId: note.userId, - blockeeId: me.id, - }); - if (block) { + const blocked = await this.userBlockingService.checkBlocked(note.userId, me.id); + if (blocked) { throw new ApiError(meta.errors.youHaveBeenBlocked); } } diff --git a/packages/backend/src/server/api/stream/types.ts b/packages/backend/src/server/api/stream/types.ts index 36bfa78363..8bb4147b43 100644 --- a/packages/backend/src/server/api/stream/types.ts +++ b/packages/backend/src/server/api/stream/types.ts @@ -25,6 +25,8 @@ export interface InternalStreamTypes { remoteUserUpdated: { id: User['id']; }; follow: { followerId: User['id']; followeeId: User['id']; }; unfollow: { followerId: User['id']; followeeId: User['id']; }; + blockingCreated: { blockerId: User['id']; blockeeId: User['id']; }; + blockingDeleted: { blockerId: User['id']; blockeeId: User['id']; }; policiesUpdated: Role['policies']; roleCreated: Role; roleDeleted: Role; From 2dfed75402969893ba10f500de254b12434c8f3f Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 4 Feb 2023 13:38:51 +0900 Subject: [PATCH 3/6] perf(server): improvement of external mediaProxy (#9787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(server): improvement of external mediaProxy * add a comment * :v: * /filesでsharpの処理を行わずリダイレクトする * fix * thumbnail => static * Fix #9788 * add avatar mode * add url * fix * static.webp * remove encodeURIComponent from media proxy path * remove existance check --- .config/example.yml | 1 + CHANGELOG.md | 3 + packages/backend/src/config.ts | 9 +++ .../backend/src/core/CustomEmojiService.ts | 2 +- .../src/core/entities/ChannelEntityService.ts | 2 +- .../core/entities/DriveFileEntityService.ts | 40 ++++++---- .../src/core/entities/UserEntityService.ts | 8 +- .../backend/src/server/FileServerService.ts | 77 +++++++++++-------- packages/backend/src/server/ServerService.ts | 2 +- .../backend/src/server/api/endpoints/meta.ts | 6 ++ .../src/server/web/UrlPreviewService.ts | 2 +- packages/frontend/src/scripts/media-proxy.ts | 20 +++-- 12 files changed, 110 insertions(+), 62 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 8fe41da15a..a19b5d04e8 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -130,6 +130,7 @@ proxyBypassHosts: #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 # Media Proxy +# Reference Implementation: https://github.com/misskey-dev/media-proxy #mediaProxy: https://example.com/proxy # Proxy remote files (default: false) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e10edf683..0ad1e36213 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ You should also include the user name that made the change. - syslogのサポートが削除されました ### Improvements +- 外部メディアプロキシへの対応を強化しました + 外部メディアプロキシのFastify実装を作りました + https://github.com/misskey-dev/media-proxy - ロールで広告の非表示が有効になっている場合は最初から広告を非表示にするように ## 13.2.6 (2023/02/01) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 1d4e700656..aa98ef1d22 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -87,6 +87,8 @@ export type Mixin = { userAgent: string; clientEntry: string; clientManifestExists: boolean; + mediaProxy: string; + externalMediaProxyEnabled: boolean; }; export type Config = Source & Mixin; @@ -135,6 +137,13 @@ export function loadConfig() { mixin.clientEntry = clientManifest['src/init.ts']; mixin.clientManifestExists = clientManifestExists; + const externalMediaProxy = config.mediaProxy ? + config.mediaProxy.endsWith('/') ? config.mediaProxy.substring(0, config.mediaProxy.length - 1) : config.mediaProxy + : null; + const internalMediaProxy = `${mixin.scheme}://${mixin.host}/proxy`; + mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy; + mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy; + if (!config.redis.prefix) config.redis.prefix = mixin.host; return Object.assign(config, mixin); diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index 39814e1be6..63f0319442 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -120,7 +120,7 @@ export class CustomEmojiService { const url = isLocal ? emojiUrl : this.config.proxyRemoteFiles - ? `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}` + ? `${this.config.mediaProxy}/emoji.webp?${query({ url: emojiUrl })}` : emojiUrl; return url; diff --git a/packages/backend/src/core/entities/ChannelEntityService.ts b/packages/backend/src/core/entities/ChannelEntityService.ts index 5e2f019a12..6ce590aa96 100644 --- a/packages/backend/src/core/entities/ChannelEntityService.ts +++ b/packages/backend/src/core/entities/ChannelEntityService.ts @@ -54,7 +54,7 @@ export class ChannelEntityService { name: channel.name, description: channel.description, userId: channel.userId, - bannerUrl: banner ? this.driveFileEntityService.getPublicUrl(banner, false) : null, + bannerUrl: banner ? this.driveFileEntityService.getPublicUrl(banner) : null, usersCount: channel.usersCount, notesCount: channel.notesCount, diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index 7f54cfdeac..efc196f74a 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -71,27 +71,41 @@ export class DriveFileEntityService { } @bindThis - public getPublicUrl(file: DriveFile, thumbnail = false): string | null { + public getPublicUrl(file: DriveFile, mode? : 'static' | 'avatar'): string | null { // static = thumbnail + const proxiedUrl = (url: string) => appendQuery( + `${this.config.mediaProxy}/${mode ?? 'image'}.webp`, + query({ + url, + ...(mode ? { [mode]: '1' } : {}), + }) + ); + // リモートかつメディアプロキシ - if (file.uri != null && file.userHost != null && this.config.mediaProxy != null) { - return appendQuery(this.config.mediaProxy, query({ - url: file.uri, - thumbnail: thumbnail ? '1' : undefined, - })); + if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) { + return proxiedUrl(file.uri); } // リモートかつ期限切れはローカルプロキシを試みる if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) { - const key = thumbnail ? file.thumbnailAccessKey : file.webpublicAccessKey; + const key = mode === 'static' ? file.thumbnailAccessKey : file.webpublicAccessKey; if (key && !key.match('/')) { // 古いものはここにオブジェクトストレージキーが入ってるので除外 - return `${this.config.url}/files/${key}`; + const url = `${this.config.url}/files/${key}`; + if (mode === 'avatar') return proxiedUrl(url); + return url; } } const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/avif', 'image/svg+xml'].includes(file.type); - return thumbnail ? (file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null)) : (file.webpublicUrl ?? file.url); + if (mode === 'static') { + return file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null); + } + + const url = file.webpublicUrl ?? file.url; + + if (mode === 'avatar') return proxiedUrl(url); + return url; } @bindThis @@ -166,8 +180,8 @@ export class DriveFileEntityService { isSensitive: file.isSensitive, blurhash: file.blurhash, properties: opts.self ? file.properties : this.getPublicProperties(file), - url: opts.self ? file.url : this.getPublicUrl(file, false), - thumbnailUrl: this.getPublicUrl(file, true), + url: opts.self ? file.url : this.getPublicUrl(file), + thumbnailUrl: this.getPublicUrl(file, 'static'), comment: file.comment, folderId: file.folderId, folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, { @@ -201,8 +215,8 @@ export class DriveFileEntityService { isSensitive: file.isSensitive, blurhash: file.blurhash, properties: opts.self ? file.properties : this.getPublicProperties(file), - url: opts.self ? file.url : this.getPublicUrl(file, false), - thumbnailUrl: this.getPublicUrl(file, true), + url: opts.self ? file.url : this.getPublicUrl(file), + thumbnailUrl: this.getPublicUrl(file, 'static'), comment: file.comment, folderId: file.folderId, folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, { diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index aaa80033b3..ff42c07359 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -314,10 +314,10 @@ export class UserEntityService implements OnModuleInit { @bindThis public async getAvatarUrl(user: User): Promise { if (user.avatar) { - return this.driveFileEntityService.getPublicUrl(user.avatar, true) ?? this.getIdenticonUrl(user.id); + return this.driveFileEntityService.getPublicUrl(user.avatar, 'avatar') ?? this.getIdenticonUrl(user.id); } else if (user.avatarId) { const avatar = await this.driveFilesRepository.findOneByOrFail({ id: user.avatarId }); - return this.driveFileEntityService.getPublicUrl(avatar, true) ?? this.getIdenticonUrl(user.id); + return this.driveFileEntityService.getPublicUrl(avatar, 'avatar') ?? this.getIdenticonUrl(user.id); } else { return this.getIdenticonUrl(user.id); } @@ -326,7 +326,7 @@ export class UserEntityService implements OnModuleInit { @bindThis public getAvatarUrlSync(user: User): string { if (user.avatar) { - return this.driveFileEntityService.getPublicUrl(user.avatar, true) ?? this.getIdenticonUrl(user.id); + return this.driveFileEntityService.getPublicUrl(user.avatar, 'avatar') ?? this.getIdenticonUrl(user.id); } else { return this.getIdenticonUrl(user.id); } @@ -422,7 +422,7 @@ export class UserEntityService implements OnModuleInit { createdAt: user.createdAt.toISOString(), updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null, lastFetchedAt: user.lastFetchedAt ? user.lastFetchedAt.toISOString() : null, - bannerUrl: user.banner ? this.driveFileEntityService.getPublicUrl(user.banner, false) : null, + bannerUrl: user.banner ? this.driveFileEntityService.getPublicUrl(user.banner) : null, bannerBlurhash: user.banner?.blurhash ?? null, isLocked: user.isLocked, isSilenced: this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 40024270ae..39bc4c1d96 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -137,38 +137,38 @@ export class FileServerService { try { if (file.state === 'remote') { - const convertFile = async () => { - if (file.fileRole === 'thumbnail') { - if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(file.mime)) { - return this.imageProcessingService.convertToWebpStream( - file.path, - 498, - 280 - ); - } else if (file.mime.startsWith('video/')) { - return await this.videoProcessingService.generateVideoThumbnail(file.path); - } - } + let image: IImageStreamable | null = null; - if (file.fileRole === 'webpublic') { - if (['image/svg+xml'].includes(file.mime)) { - return this.imageProcessingService.convertToWebpStream( - file.path, - 2048, - 2048, - { ...webpDefault, lossless: true } - ) - } - } + if (file.fileRole === 'thumbnail') { + if (isMimeImage(file.mime, 'sharp-convertible-image')) { + reply.header('Cache-Control', 'max-age=31536000, immutable'); - return { + const url = new URL(`${this.config.mediaProxy}/static.webp`); + url.searchParams.set('url', file.url); + url.searchParams.set('static', '1'); + return await reply.redirect(301, url.toString()); + } else if (file.mime.startsWith('video/')) { + image = await this.videoProcessingService.generateVideoThumbnail(file.path); + } + } + + if (file.fileRole === 'webpublic') { + if (['image/svg+xml'].includes(file.mime)) { + reply.header('Cache-Control', 'max-age=31536000, immutable'); + + const url = new URL(`${this.config.mediaProxy}/svg.webp`); + url.searchParams.set('url', file.url); + return await reply.redirect(301, url.toString()); + } + } + + if (!image) { + image = { data: fs.createReadStream(file.path), ext: file.ext, type: file.mime, }; - }; - - const image = await convertFile(); + } if ('pipe' in image.data && typeof image.data.pipe === 'function') { // image.dataがstreamなら、stream終了後にcleanup @@ -180,7 +180,6 @@ export class FileServerService { } reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(image.type) ? image.type : 'application/octet-stream'); - reply.header('Cache-Control', 'max-age=31536000, immutable'); return image.data; } @@ -217,6 +216,23 @@ export class FileServerService { return; } + if (this.config.externalMediaProxyEnabled) { + // 外部のメディアプロキシが有効なら、そちらにリダイレクト + + reply.header('Cache-Control', 'public, max-age=259200'); // 3 days + + const url = new URL(`${this.config.mediaProxy}/${request.params.url || ''}`); + + for (const [key, value] of Object.entries(request.query)) { + url.searchParams.append(key, value); + } + + return await reply.redirect( + 301, + url.toString(), + ); + } + // Create temp file const file = await this.getStreamAndTypeFromUrl(url); if (file === '404') { @@ -236,7 +252,7 @@ export class FileServerService { const isAnimationConvertibleImage = isMimeImage(file.mime, 'sharp-animation-convertible-image'); let image: IImageStreamable | null = null; - if ('emoji' in request.query && isConvertibleImage) { + if (('emoji' in request.query || 'avatar' in request.query) && isConvertibleImage) { if (!isAnimationConvertibleImage && !('static' in request.query)) { image = { data: fs.createReadStream(file.path), @@ -246,7 +262,7 @@ export class FileServerService { } else { const data = sharp(file.path, { animated: !('static' in request.query) }) .resize({ - height: 128, + height: 'emoji' in request.query ? 128 : 320, withoutEnlargement: true, }) .webp(webpDefault); @@ -370,7 +386,7 @@ export class FileServerService { @bindThis private async getFileFromKey(key: string): Promise< - { state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; mime: string; ext: string | null; path: string; cleanup: () => void; } + { state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; url: string; mime: string; ext: string | null; path: string; cleanup: () => void; } | { state: 'stored_internal'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; mime: string; ext: string | null; path: string; } | '404' | '204' @@ -392,6 +408,7 @@ export class FileServerService { const result = await this.downloadAndDetectTypeFromUrl(file.uri); return { ...result, + url: file.uri, fileRole: isThumbnail ? 'thumbnail' : isWebpublic ? 'webpublic' : 'original', file, } diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index beb3a34ecd..c7a2c99f94 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -106,7 +106,7 @@ export class ServerService { } } - const url = new URL('/proxy/emoji.webp', this.config.url); + const url = new URL(`${this.config.mediaProxy}/emoji.webp`); // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） url.searchParams.set('url', emoji.publicUrl || emoji.originalUrl); url.searchParams.set('emoji', '1'); diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 3baf945323..2fa7a09d49 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -181,6 +181,10 @@ export const meta = { type: 'string', optional: false, nullable: true, }, + mediaProxy: { + type: 'string', + optional: false, nullable: false, + }, features: { type: 'object', optional: true, nullable: false, @@ -307,6 +311,8 @@ export default class extends Endpoint { policies: { ...DEFAULT_POLICIES, ...instance.policies }, + mediaProxy: this.config.mediaProxy, + ...(ps.detail ? { pinnedPages: instance.pinnedPages, pinnedClipId: instance.pinnedClipId, diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 802b404ce6..1bf88fe434 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -33,7 +33,7 @@ export class UrlPreviewService { private wrap(url?: string): string | null { return url != null ? url.match(/^https?:\/\//) - ? `${this.config.url}/proxy/preview.webp?${query({ + ? `${this.config.mediaProxy}/preview.webp?${query({ url, preview: '1', })}` diff --git a/packages/frontend/src/scripts/media-proxy.ts b/packages/frontend/src/scripts/media-proxy.ts index bea164e7c8..274e96e0a1 100644 --- a/packages/frontend/src/scripts/media-proxy.ts +++ b/packages/frontend/src/scripts/media-proxy.ts @@ -1,8 +1,9 @@ import { query, appendQuery } from '@/scripts/url'; import { url } from '@/config'; +import { instance } from '@/instance'; export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string { - if (imageUrl.startsWith(`${url}/proxy/`) || imageUrl.startsWith('/proxy/')) { + if (imageUrl.startsWith(instance.mediaProxy + '/') || imageUrl.startsWith('/proxy/')) { // もう既にproxyっぽそうだったらsearchParams付けるだけ return appendQuery(imageUrl, query({ fallback: '1', @@ -10,7 +11,7 @@ export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string { })); } - return `${url}/proxy/image.webp?${query({ + return `${instance.mediaProxy}/image.webp?${query({ url: imageUrl, fallback: '1', ...(type ? { [type]: '1' } : {}), @@ -25,22 +26,19 @@ export function getProxiedImageUrlNullable(imageUrl: string | null | undefined, export function getStaticImageUrl(baseUrl: string): string { const u = baseUrl.startsWith('http') ? new URL(baseUrl) : new URL(baseUrl, url); - if (u.href.startsWith(`${url}/proxy/`)) { - // もう既にproxyっぽそうだったらsearchParams付けるだけ - u.searchParams.set('static', '1'); - return u.href; - } - if (u.href.startsWith(`${url}/emoji/`)) { // もう既にemojiっぽそうだったらsearchParams付けるだけ u.searchParams.set('static', '1'); return u.href; } - // 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する - const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`; + if (u.href.startsWith(instance.mediaProxy + '/')) { + // もう既にproxyっぽそうだったらsearchParams付けるだけ + u.searchParams.set('static', '1'); + return u.href; + } - return `${url}/proxy/${dummy}?${query({ + return `${instance.mediaProxy}/static.webp?${query({ url: u.href, static: '1', })}`; From 38f9d1e76428bea47c5944c440eab25428c7d99e Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 4 Feb 2023 14:20:07 +0900 Subject: [PATCH 4/6] fix(client): validate urls to improve security --- CHANGELOG.md | 7 +++++++ packages/backend/src/server/web/UrlPreviewService.ts | 8 ++++++++ packages/frontend/src/components/MkUrlPreview.vue | 3 ++- packages/frontend/src/components/MkYoutubePlayer.vue | 3 ++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad1e36213..66382cac12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ You should also include the user name that made the change. --> +## 13.x.x (unreleased) + +### Improvements + +### Bugfixes +- Client: validate urls to improve security + ## 13.3.1 (2023/02/04) ### Bugfixes diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 1bf88fe434..57461b7a33 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -73,6 +73,14 @@ export class UrlPreviewService { }); this.logger.succ(`Got preview of ${url}: ${summary.title}`); + + if (summary.url && !(summary.url.startsWith('http://') || summary.url.startsWith('https://'))) { + throw new Error('unsupported schema included'); + } + + if (summary.player?.url && !(summary.player.url.startsWith('http://') || summary.player.url.startsWith('https://'))) { + throw new Error('unsupported schema included'); + } summary.icon = this.wrap(summary.icon); summary.thumbnail = this.wrap(summary.thumbnail); diff --git a/packages/frontend/src/components/MkUrlPreview.vue b/packages/frontend/src/components/MkUrlPreview.vue index f7677faf74..62e58e1553 100644 --- a/packages/frontend/src/components/MkUrlPreview.vue +++ b/packages/frontend/src/components/MkUrlPreview.vue @@ -1,7 +1,8 @@