diff --git a/.config/docker_example.yml b/.config/docker_example.yml index de95f1b21a..cc9738657e 100755 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -163,6 +163,16 @@ redis: # #prefix: example-prefix # #db: 1 +#redisForRemoteClips: +# host: localhost +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 +# # You can specify more ioredis options... +# #username: example-username + # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/.config/example.yml b/.config/example.yml index 21e85b7b89..ab44e5baf5 100755 --- a/.config/example.yml +++ b/.config/example.yml @@ -172,6 +172,16 @@ redis: # # You can specify more ioredis options... # #username: example-username +#redisForRemoteClips: +# host: localhost +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 +# # You can specify more ioredis options... +# #username: example-username + # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/CHANGELOG_yojo.md b/CHANGELOG_yojo.md new file mode 100644 index 0000000000..86f9254a10 --- /dev/null +++ b/CHANGELOG_yojo.md @@ -0,0 +1,97 @@ + +## 0.3.0 (unreleased) + +### Release Date + +### General +- Feat: リモートユーザーのクリップが閲覧できるように + - お気に入りは未実装です + - ログインが必要な投稿は見れません + +### Client +- + +### Server +- Feat: OpenSearchを利用できるように +- Enhance: 高度な検索に新たな条件を追加(OpenSearchが必要です) + - 添付ファイルのセンシティブ条件(なし/含む/除外) + - 引用ノート除外 + - 検索方法の詳細はdoc/Advanced-Search.mdに +- Change:APIのパラメータを変更 + - notes/advanced-search の"excludeNsfw"を"excludeCW"に変更 + - notes/advanced-search の"channelId"を削除 + +## 0.2.2 +Cherrypick 4.9.0-beta.2 + +### General + +### Client + +### Server +- remove: チャンネル機能のAPIを削除 + +## 0.2.1 +Cherrypick 4.9.0-beta.2 + +### Client +- feat: マスコット画像を表示するウィジェットを追加 + +## 0.2.0 +Cherrypick 4.9.0-beta.2 + +### General +- enhance: ノートとユーザーの検索時に照会を行うかが選択できるようになりました + - @foo@example.com 形式でユーザ検索した場合に照会ができるようになりました +- remove: チャンネル機能を削除 +- add: 公式タグ機能を追加 + - インスタンスで利用が推奨されるハッシュタグの一覧です + +### Client +- feat: 未ログイン時サーバーで指定されている場合、マスコット画像が表示されます +- enhance: ハッシュタグTLをリアルタイムに更新 +- fix: アンケート選択肢にリモート絵文字を表示 + +### Server +- fix: リモートユーザーにはファイルサイズ制限を適用しない +- fix: メディアタイムライン選択時の上部のアイコンを修正 +- add: メディアタイムラインのチュートリアルを追加 +- feat: マスコット画像を指定できるように(コントロールパネル/ブランディング) + - デフォルト値(/assets/ai.png)または空欄の場合タイムラインが表示されます + +## 0.1.0 (unreleased) +Cherrypick 4.8.0 + +### General +- enhance: メディアプロキシurlと拡大画像urlを分割 +- enhance: 1ファイルの容量をロールでも制限できるように + +### Client +- enhance: ノートとユーザーの検索時に照会を行うかが選択できるようになりました + - @foo​@example.com 形式でユーザ検索した場合に照会ができるようになりました +- add: 通知音を追加 [@mujin-nohuman (無人)](https://github.com/mujin-nohuman) +- fix: "キャッシュをクリア"してもインスタンス情報が更新されない不具合を修正 [#101](https://github.com/yojo-art/cherrypick/issues/101) + +### Server +- enhance: remoteProxyエンドポイント設定を追加 +- fix: webpublic生成時にドライブの縮小設定を見るように + +### Others +- engawaをマージ +- cherrypickからフォーク diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 09971e8ca0..02f6de546c 100755 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -77,12 +77,17 @@ const $redisForTimelines: Provider = { }, inject: [DI.config], }; +const $redisForRemoteClips: Provider = { + provide: DI.redisForRemoteClips, + useFactory: (config: Config) => { + return new Redis.Redis(config.redisForRemoteClips); + }, + inject: [DI.config], +}; @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines], - exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( @@ -91,6 +96,7 @@ export class GlobalModule implements OnApplicationShutdown { @Inject(DI.redisForPub) private redisForPub: Redis.Redis, @Inject(DI.redisForSub) private redisForSub: Redis.Redis, @Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis, + @Inject(DI.redisForRemoteClips) private redisForRemoteClips: Redis.Redis, ) { } public async dispose(): Promise { @@ -103,6 +109,7 @@ export class GlobalModule implements OnApplicationShutdown { this.redisForPub.disconnect(), this.redisForSub.disconnect(), this.redisForTimelines.disconnect(), + this.redisForRemoteClips.disconnect(), ]); } diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 3f9967ad19..a50f3a0cba 100755 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -51,6 +51,7 @@ type Source = { redisForPubsub?: RedisOptionsSource; redisForJobQueue?: RedisOptionsSource; redisForTimelines?: RedisOptionsSource; + redisForRemoteClips?: RedisOptionsSource; meilisearch?: { host: string; port: string; @@ -186,6 +187,7 @@ export type Config = { redisForPubsub: RedisOptions & RedisOptionsSource; redisForJobQueue: RedisOptions & RedisOptionsSource; redisForTimelines: RedisOptions & RedisOptionsSource; + redisForRemoteClips: RedisOptions & RedisOptionsSource; sentryForBackend: { options: Partial; enableNodeProfiling: boolean; } | undefined; sentryForFrontend: { options: Partial } | undefined; perChannelMaxNoteCacheCount: number; @@ -282,6 +284,7 @@ export function loadConfig(): Config { redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis, redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis, redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis, + redisForRemoteClips: config.redisForRemoteClips ? convertRedisOptions(config.redisForRemoteClips, host) : redis, sentryForBackend: config.sentryForBackend, sentryForFrontend: config.sentryForFrontend, id: config.id, diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index d4a21ab625..5e6eecfc3b 100755 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -11,6 +11,7 @@ export const DI = { redisForPub: Symbol('redisForPub'), redisForSub: Symbol('redisForSub'), redisForTimelines: Symbol('redisForTimelines'), + redisForRemoteClips: Symbol('redisForRemoteClips'), //#region Repositories usersRepository: Symbol('usersRepository'), diff --git a/packages/backend/src/server/HealthServerService.ts b/packages/backend/src/server/HealthServerService.ts index 2c3ed85925..6b59d18e2a 100755 --- a/packages/backend/src/server/HealthServerService.ts +++ b/packages/backend/src/server/HealthServerService.ts @@ -27,6 +27,9 @@ export class HealthServerService { @Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis, + @Inject(DI.redisForRemoteClips) + private redisForRemoteClips: Redis.Redis, + @Inject(DI.db) private db: DataSource, @@ -43,6 +46,7 @@ export class HealthServerService { this.redisForPub.ping(), this.redisForSub.ping(), this.redisForTimelines.ping(), + this.redisForRemoteClips.ping(), this.db.query('SELECT 1'), ...(this.meilisearch ? [this.meilisearch.health()] : []), ]).then(() => 200, () => 503)); diff --git a/packages/backend/src/server/api/endpoints/clips/favorite.ts b/packages/backend/src/server/api/endpoints/clips/favorite.ts index 11f8ec3e92..a73f0f892d 100755 --- a/packages/backend/src/server/api/endpoints/clips/favorite.ts +++ b/packages/backend/src/server/api/endpoints/clips/favorite.ts @@ -31,13 +31,18 @@ export const meta = { code: 'ALREADY_FAVORITED', id: '92658936-c625-4273-8326-2d790129256e', }, + unimplemented: { + message: 'Unimplemented.', + code: 'UNIMPLEMENTED', + id: '37561aed-4ba4-4a53-9efe-a0aa255e9bb3', + }, }, } as const; export const paramDef = { type: 'object', properties: { - clipId: { type: 'string', format: 'misskey:id' }, + clipId: { type: 'string' }, }, required: ['clipId'], } as const; @@ -54,6 +59,9 @@ export default class extends Endpoint { // eslint- private idService: IdService, ) { super(meta, paramDef, async (ps, me) => { + if (ps.clipId.split('@').length > 1) { + throw new ApiError(meta.errors.unimplemented); + } const clip = await this.clipsRepository.findOneBy({ id: ps.clipId }); if (clip == null) { throw new ApiError(meta.errors.noSuchClip); diff --git a/packages/backend/src/server/api/endpoints/clips/notes.ts b/packages/backend/src/server/api/endpoints/clips/notes.ts index 943c31c894..7f79a14671 100755 --- a/packages/backend/src/server/api/endpoints/clips/notes.ts +++ b/packages/backend/src/server/api/endpoints/clips/notes.ts @@ -4,11 +4,23 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import got, * as Got from 'got'; +import * as Redis from 'ioredis'; +import type { Config } from '@/config.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { NotesRepository, ClipsRepository, ClipNotesRepository } from '@/models/_.js'; +import type { NotesRepository, ClipsRepository, ClipNotesRepository, MiNote } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { awaitAll } from '@/misc/prelude/await-all.js'; +import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; +import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; +import { IObject } from '@/core/activitypub/type.js'; +import { MetaService } from '@/core/MetaService.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { ApLoggerService } from '@/core/activitypub/ApLoggerService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -24,6 +36,16 @@ export const meta = { code: 'NO_SUCH_CLIP', id: '1d7645e6-2b6d-4635-b0fe-fe22b0e72e00', }, + invalidIdFormat: { + message: 'Invalid id format.', + code: 'INVALID_ID_FORMAT', + id: '42d11fe5-686e-41ee-9e7f-e804ec3a388d', + }, + failedToResolveRemoteUser: { + message: 'failedToResolveRemoteUser.', + code: 'FAILED_TO_RESOLVE_REMOTE_USER', + id: 'af0ecffc-8717-409e-a8d4-e1e3a5d2497f', + }, }, res: { @@ -40,7 +62,7 @@ export const meta = { export const paramDef = { type: 'object', properties: { - clipId: { type: 'string', format: 'misskey:id' }, + clipId: { type: 'string' }, limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, @@ -51,6 +73,8 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( + @Inject(DI.config) + private config: Config, @Inject(DI.clipsRepository) private clipsRepository: ClipsRepository, @@ -59,43 +83,189 @@ export default class extends Endpoint { // eslint- @Inject(DI.clipNotesRepository) private clipNotesRepository: ClipNotesRepository, + @Inject(DI.redisForRemoteClips) + private redisForRemoteClips: Redis.Redis, + + private httpRequestService: HttpRequestService, + private apNoteService: ApNoteService, + private metaService: MetaService, + private utilityService: UtilityService, private noteEntityService: NoteEntityService, private queryService: QueryService, + private apLoggerService: ApLoggerService, ) { super(meta, paramDef, async (ps, me) => { - const clip = await this.clipsRepository.findOneBy({ - id: ps.clipId, - }); + const parsed_id = ps.clipId.split('@'); + let notes = []; + if (parsed_id.length === 2 ) {//is remote + const url = 'https://' + parsed_id[1] + '/api/clips/notes'; + apLoggerService.logger.debug('remote clip ' + url); + notes = await remote(config, httpRequestService, redisForRemoteClips, apNoteService, metaService, utilityService, apLoggerService, url, parsed_id[0], parsed_id[1], ps.clipId, ps.limit, ps.sinceId, ps.untilId); + } else if (parsed_id.length === 1 ) {//is not local + const clip = await this.clipsRepository.findOneBy({ + id: ps.clipId, + }); - if (clip == null) { - throw new ApiError(meta.errors.noSuchClip); + if (clip == null) { + throw new ApiError(meta.errors.noSuchClip); + } + + if (!clip.isPublic && (me == null || (clip.userId !== me.id))) { + throw new ApiError(meta.errors.noSuchClip); + } + + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + .innerJoin(this.clipNotesRepository.metadata.targetName, 'clipNote', 'clipNote.noteId = note.id') + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .andWhere('clipNote.clipId = :clipId', { clipId: clip.id }); + + if (me) { + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); + } + + notes = await query + .limit(ps.limit) + .getMany(); + } else { + throw new ApiError(meta.errors.invalidIdFormat); } - if (!clip.isPublic && (me == null || (clip.userId !== me.id))) { - throw new ApiError(meta.errors.noSuchClip); - } - - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) - .innerJoin(this.clipNotesRepository.metadata.targetName, 'clipNote', 'clipNote.noteId = note.id') - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .andWhere('clipNote.clipId = :clipId', { clipId: clip.id }); - - if (me) { - this.queryService.generateVisibilityQuery(query, me); - this.queryService.generateMutedUserQuery(query, me); - this.queryService.generateBlockedUserQuery(query, me); - } - - const notes = await query - .limit(ps.limit) - .getMany(); - return await this.noteEntityService.packMany(notes, me); }); } } + +async function remote( + config:Config, + httpRequestService: HttpRequestService, + redisForRemoteClips: Redis.Redis, + apNoteService: ApNoteService, + metaService: MetaService, + utilityService: UtilityService, + apLoggerService: ApLoggerService, + url:string, + clipId:string, + host:string, + local_id:string, + limit:number, + sinceId:string|undefined, + untilId:string|undefined, +) { + const cache_key = local_id + '-' + (sinceId ? sinceId : '') + '-' + (untilId ? untilId : '') + limit; + const cache_value = await redisForRemoteClips.get(cache_key); + let remote_json = null; + if (cache_value === null) { + const sinceIdRemote = sinceId ? await redisForRemoteClips.get(sinceId + '@' + host) : undefined; + const untilIdRemote = untilId ? await redisForRemoteClips.get(untilId + '@' + host) : undefined; + const timeout = 30 * 1000; + const operationTimeout = 60 * 1000; + const res = got.post(url, { + headers: { + 'User-Agent': config.userAgent, + 'Content-Type': 'application/json; charset=utf-8', + }, + timeout: { + lookup: timeout, + connect: timeout, + secureConnect: timeout, + socket: timeout, // read timeout + response: timeout, + send: timeout, + request: operationTimeout, // whole operation timeout + }, + agent: { + http: httpRequestService.httpAgent, + https: httpRequestService.httpsAgent, + }, + http2: true, + retry: { + limit: 1, + }, + enableUnixSockets: false, + body: JSON.stringify({ + clipId, + limit, + sinceId: sinceIdRemote, + untilId: untilIdRemote, + }), + }); + remote_json = await res.text(); + redisForRemoteClips.set(cache_key, remote_json); + redisForRemoteClips.expire(cache_key, 10 * 60); + } else { + remote_json = cache_value; + } + const remote_notes = JSON.parse(remote_json); + //リモートに照会する回数の上限 + const create_limit = 5; + let create_count = 0; + const notes = []; + for (const note of remote_notes) { + const uri = note.uri ? note.uri : 'https://' + host + '/notes/' + note.id; + if (uri !== null) { + if (create_count > create_limit) { + break; + } + const local_note = await remoteNote(apNoteService, uri, redisForRemoteClips, metaService, utilityService, apLoggerService, host, note.id); + if (local_note !== null) { + if (local_note.is_create) { + create_count++; + } + notes.push(local_note.note); + } + } + } + return notes; +} + +class RemoteNote { + note:MiNote; + is_create:boolean; +} + +async function remoteNote( + apNoteService: ApNoteService, + uri: string, + redisForRemoteClips: Redis.Redis, + metaService: MetaService, + utilityService: UtilityService, + apLoggerService: ApLoggerService, + host:string, + remote_note_id:string, +): Promise { + const fetchedMeta = await metaService.fetch(); + apLoggerService.logger.debug('remote clip note fetch ' + uri); + let note; + let is_create = false; + try { + //ブロックされたインスタンスの投稿は無かった事にする + if (utilityService.isBlockedHost(fetchedMeta.blockedHosts, utilityService.extractDbHost(uri))) return null; + //ローカルのDBから取得を試みる + note = await apNoteService.fetchNote(uri); + if (note == null) { + //ダメそうなら照会 + note = await apNoteService.createNote(uri, undefined, true); + is_create = true; + } + } catch (e) { + apLoggerService.logger.warn(String(e)); + //照会失敗した時はクリップ内に無かった事にする + return null; + } + if (note !== null) { + redisForRemoteClips.set(note.id + '@' + host, remote_note_id); + return { + note, + is_create, + }; + } else { + return null; + } +} diff --git a/packages/backend/src/server/api/endpoints/clips/show.ts b/packages/backend/src/server/api/endpoints/clips/show.ts index 1078a1b176..91a91b8997 100755 --- a/packages/backend/src/server/api/endpoints/clips/show.ts +++ b/packages/backend/src/server/api/endpoints/clips/show.ts @@ -4,10 +4,17 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import got, * as Got from 'got'; +import * as Redis from 'ioredis'; import { Endpoint } from '@/server/api/endpoint-base.js'; import type { ClipsRepository } from '@/models/_.js'; import { ClipEntityService } from '@/core/entities/ClipEntityService.js'; import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { awaitAll } from '@/misc/prelude/await-all.js'; +import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -23,6 +30,16 @@ export const meta = { code: 'NO_SUCH_CLIP', id: 'c3c5fe33-d62c-44d2-9ea5-d997703f5c20', }, + invalidIdFormat: { + message: 'Invalid id format.', + code: 'INVALID_ID_FORMAT', + id: 'df45c7d1-cd15-4a35-b3e1-8c9f987c4f5c', + }, + failedToResolveRemoteUser: { + message: 'failedToResolveRemoteUser.', + code: 'FAILED_TO_RESOLVE_REMOTE_USER', + id: '56d5e552-d55a-47e3-9f37-6dc85a93ecf9', + }, }, res: { @@ -35,7 +52,7 @@ export const meta = { export const paramDef = { type: 'object', properties: { - clipId: { type: 'string', format: 'misskey:id' }, + clipId: { type: 'string' }, }, required: ['clipId'], } as const; @@ -43,12 +60,28 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( + @Inject(DI.config) + private config: Config, @Inject(DI.clipsRepository) private clipsRepository: ClipsRepository, + @Inject(DI.redisForRemoteClips) + private redisForRemoteClips: Redis.Redis, + private httpRequestService: HttpRequestService, + private userEntityService: UserEntityService, + private remoteUserResolveService: RemoteUserResolveService, private clipEntityService: ClipEntityService, ) { super(meta, paramDef, async (ps, me) => { + const parsed_id = ps.clipId.split('@'); + if (parsed_id.length === 2 ) {//is remote + const url = 'https://' + parsed_id[1] + '/api/clips/show'; + console.log(url); + return remote(config, httpRequestService, userEntityService, remoteUserResolveService, redisForRemoteClips, url, parsed_id[0], parsed_id[1], ps.clipId); + } + if (parsed_id.length !== 1 ) {//is not local + throw new ApiError(meta.errors.invalidIdFormat); + } // Fetch the clip const clip = await this.clipsRepository.findOneBy({ id: ps.clipId, @@ -66,3 +99,75 @@ export default class extends Endpoint { // eslint- }); } } + +async function remote( + config:Config, + httpRequestService: HttpRequestService, + userEntityService: UserEntityService, + remoteUserResolveService: RemoteUserResolveService, + redisForRemoteClips: Redis.Redis, + url:string, + clipId:string, + host:string, + local_id:string, +) { + const cache_key = local_id + '-info'; + const cache_value = await redisForRemoteClips.get(cache_key); + let remote_json = null; + if (cache_value === null) { + const timeout = 30 * 1000; + const operationTimeout = 60 * 1000; + const res = got.post(url, { + headers: { + 'User-Agent': config.userAgent, + 'Content-Type': 'application/json; charset=utf-8', + }, + timeout: { + lookup: timeout, + connect: timeout, + secureConnect: timeout, + socket: timeout, // read timeout + response: timeout, + send: timeout, + request: operationTimeout, // whole operation timeout + }, + agent: { + http: httpRequestService.httpAgent, + https: httpRequestService.httpsAgent, + }, + http2: true, + retry: { + limit: 1, + }, + enableUnixSockets: false, + body: JSON.stringify({ + clipId, + }), + }); + remote_json = await res.text(); + redisForRemoteClips.set(cache_key, remote_json); + redisForRemoteClips.expire(cache_key, 10 * 60); + } else { + remote_json = cache_value; + } + const remote_clip = JSON.parse(remote_json); + if (remote_clip.user == null || remote_clip.user.username == null) { + throw new ApiError(meta.errors.failedToResolveRemoteUser); + } + const user = await remoteUserResolveService.resolveUser(remote_clip.user.username, host).catch(err => { + throw new ApiError(meta.errors.failedToResolveRemoteUser); + }); + return await awaitAll({ + id: local_id, + createdAt: remote_clip.createdAt ? remote_clip.createdAt : null, + lastClippedAt: remote_clip.lastClippedAt ? remote_clip.lastClippedAt : null, + userId: user.id, + user: userEntityService.pack(user), + name: remote_clip.name, + description: remote_clip.description, + isPublic: true, + favoritedCount: remote_clip.favoritedCount, + isFavorited: false, + notesCount: remote_clip.notesCount, + }); +} diff --git a/packages/backend/src/server/api/endpoints/users/clips.ts b/packages/backend/src/server/api/endpoints/users/clips.ts index 7f7d2ea8cc..4bc6d41d35 100755 --- a/packages/backend/src/server/api/endpoints/users/clips.ts +++ b/packages/backend/src/server/api/endpoints/users/clips.ts @@ -4,11 +4,18 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import type { ClipsRepository } from '@/models/_.js'; +import Redis from 'ioredis'; +import got, * as Got from 'got'; +import type { ClipsRepository, MiUser, UsersRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import { ClipEntityService } from '@/core/entities/ClipEntityService.js'; import { DI } from '@/di-symbols.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import type { Config } from '@/config.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { awaitAll } from '@/misc/prelude/await-all.js'; +import type { FindOptionsWhere } from 'typeorm'; export const meta = { tags: ['users', 'clips'], @@ -31,8 +38,8 @@ export const paramDef = { properties: { userId: { type: 'string', format: 'misskey:id' }, limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, - sinceId: { type: 'string', format: 'misskey:id' }, - untilId: { type: 'string', format: 'misskey:id' }, + sinceId: { type: 'string' }, + untilId: { type: 'string' }, }, required: ['userId'], } as const; @@ -40,13 +47,28 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( + @Inject(DI.config) + private config: Config, @Inject(DI.clipsRepository) private clipsRepository: ClipsRepository, + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + @Inject(DI.redisForRemoteClips) + private redisForRemoteClips: Redis.Redis, + private httpRequestService: HttpRequestService, + private userEntityService: UserEntityService, private clipEntityService: ClipEntityService, private queryService: QueryService, ) { super(meta, paramDef, async (ps, me) => { + const q: FindOptionsWhere = { id: ps.userId }; + + const user = await this.usersRepository.findOneBy(q); + if (user === null) return []; + if (userEntityService.isRemoteUser(user)) { + return remote(config, httpRequestService, redisForRemoteClips, userEntityService, user, ps.limit, ps.sinceId, ps.untilId); + } const query = this.queryService.makePaginationQuery(this.clipsRepository.createQueryBuilder('clip'), ps.sinceId, ps.untilId) .andWhere('clip.userId = :userId', { userId: ps.userId }) .andWhere('clip.isPublic = true'); @@ -59,3 +81,165 @@ export default class extends Endpoint { // eslint- }); } } + +async function remote( + config:Config, + httpRequestService: HttpRequestService, + redisForRemoteClips: Redis.Redis, + userEntityService: UserEntityService, + user:MiUser, + limit:number, + sinceId:string|undefined, + untilId:string|undefined, +) { + const cache_key = user.id + '-' + sinceId + '-' + untilId + '-clips'; + const cache_value = await redisForRemoteClips.get(cache_key); + if (cache_value !== null) { + //ステータス格納 + if (cache_value.startsWith('__')) { + if (cache_value === '__SKIP_FETCH') return []; + //未定義のステータス + return []; + } + return JSON.parse(cache_value); + } + if (user.host == null) { + //ローカルユーザーではない + return []; + } + const remote_user_id = await fetch_remote_user(config, httpRequestService, redisForRemoteClips, user.username, user.host); + if (remote_user_id === null) { + return []; + } + const remote_json = await fetch_remote_clip(config, httpRequestService, remote_user_id, user.host, limit, sinceId, untilId); + const json = JSON.parse(remote_json); + const clips = []; + for (const remote_clip of json) { + const clip = await awaitAll({ + id: remote_clip.id + '@' + user.host, + createdAt: remote_clip.createdAt ? remote_clip.createdAt : null, + lastClippedAt: remote_clip.lastClippedAt ? remote_clip.lastClippedAt : null, + userId: user.id, + user: userEntityService.pack(user), + name: remote_clip.name, + description: remote_clip.description, + isPublic: true, + favoritedCount: remote_clip.favoritedCount, + isFavorited: false, + notesCount: remote_clip.notesCount, + }); + clips.push(clip); + } + await redisForRemoteClips.set(cache_key, JSON.stringify(clips)); + await redisForRemoteClips.expire(cache_key, 10 * 60); + return clips; +} + +async function fetch_remote_user( + config:Config, + httpRequestService: HttpRequestService, + redisForRemoteClips: Redis.Redis, + username:string, + host:string, +) { + const cache_key = username + '@' + host + '-userid'; + const id = await redisForRemoteClips.get(cache_key); + if (id !== null) { + if (id === '__NOT_MISSKEY') { + return null; + } + if (id === '__INTERVAL') { + return null; + } + return id; + } + try { + const url = 'https://' + host + '/api/users/show'; + const timeout = 30 * 1000; + const operationTimeout = 60 * 1000; + const res = got.post(url, { + headers: { + 'User-Agent': config.userAgent, + 'Content-Type': 'application/json; charset=utf-8', + }, + timeout: { + lookup: timeout, + connect: timeout, + secureConnect: timeout, + socket: timeout, // read timeout + response: timeout, + send: timeout, + request: operationTimeout, // whole operation timeout + }, + agent: { + http: httpRequestService.httpAgent, + https: httpRequestService.httpsAgent, + }, + http2: true, + retry: { + limit: 1, + }, + enableUnixSockets: false, + body: JSON.stringify({ + username, + }), + }); + const text = await res.text(); + const json = JSON.parse(text); + if (json.id != null) { + redisForRemoteClips.set(cache_key, json.id); + return json.id as string; + } + } catch { + redisForRemoteClips.set(cache_key, '__INTERVAL'); + await redisForRemoteClips.expire(cache_key, 60 * 60); + } + return null; +} + +async function fetch_remote_clip( + config:Config, + httpRequestService: HttpRequestService, + userId:string, + host:string, + limit:number, + sinceId:string|undefined, + untilId:string|undefined, +) { + const url = 'https://' + host + '/api/users/clips'; + const sinceIdRemote = sinceId ? sinceId.split('@')[0] : undefined; + const untilIdRemote = untilId ? untilId.split('@')[0] : undefined; + const timeout = 30 * 1000; + const operationTimeout = 60 * 1000; + const res = got.post(url, { + headers: { + 'User-Agent': config.userAgent, + 'Content-Type': 'application/json; charset=utf-8', + }, + timeout: { + lookup: timeout, + connect: timeout, + secureConnect: timeout, + socket: timeout, // read timeout + response: timeout, + send: timeout, + request: operationTimeout, // whole operation timeout + }, + agent: { + http: httpRequestService.httpAgent, + https: httpRequestService.httpsAgent, + }, + http2: true, + retry: { + limit: 1, + }, + enableUnixSockets: false, + body: JSON.stringify({ + userId, + limit, + sinceId: sinceIdRemote, + untilId: untilIdRemote, + }), + }); + return await res.text(); +} diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index b443da4882..479339d5e4 100755 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -13737,7 +13737,6 @@ export type operations = { requestBody: { content: { 'application/json': { - /** Format: misskey:id */ clipId: string; /** @default 10 */ limit?: number; @@ -13797,7 +13796,6 @@ export type operations = { requestBody: { content: { 'application/json': { - /** Format: misskey:id */ clipId: string; }; }; @@ -13908,7 +13906,6 @@ export type operations = { requestBody: { content: { 'application/json': { - /** Format: misskey:id */ clipId: string; }; }; @@ -26026,9 +26023,7 @@ export type operations = { userId: string; /** @default 10 */ limit?: number; - /** Format: misskey:id */ sinceId?: string; - /** Format: misskey:id */ untilId?: string; }; };