diff --git a/src/modules/library/Library.types.ts b/src/modules/library/Library.types.ts index 183cc8cb..cb8763be 100644 --- a/src/modules/library/Library.types.ts +++ b/src/modules/library/Library.types.ts @@ -55,3 +55,8 @@ export type LibraryDuplicatesWorkerInput = { + mangasToCheck: Manga[]; + mangas: Manga[]; +}; diff --git a/src/modules/library/util/LibraryDuplicates.util.ts b/src/modules/library/util/LibraryDuplicates.util.ts index afb200b0..dbd3af1b 100644 --- a/src/modules/library/util/LibraryDuplicates.util.ts +++ b/src/modules/library/util/LibraryDuplicates.util.ts @@ -60,12 +60,13 @@ const findDuplicatesByTitleAndAlternativeTitlesSingleManga = ( - mangas: Manga[], -): Promise> => { + mangasToCheck: Manga[], + mangas: Manga[] = mangasToCheck, +): TMangaDuplicates => { const titleToMangas: TMangaDuplicates = {}; const titleToAlternativeTitleMatches: TMangaDuplicates = {}; - mangas.forEach((mangaToCheck) => { + mangasToCheck.forEach((mangaToCheck) => { const titleToCheck = enhancedCleanup(mangaToCheck.title); titleToMangas[titleToCheck] ??= []; diff --git a/src/modules/library/workers/LibraryDuplicatesDescriptionWorker.ts b/src/modules/library/workers/LibraryDuplicatesDescriptionWorker.ts new file mode 100644 index 00000000..3d10d4ec --- /dev/null +++ b/src/modules/library/workers/LibraryDuplicatesDescriptionWorker.ts @@ -0,0 +1,17 @@ +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { findDuplicatesByTitleAndAlternativeTitles } from '@/modules/library/util/LibraryDuplicates.util.ts'; +import { LibraryDuplicatesDescriptionWorkerInput } from '@/modules/library/Library.types.ts'; + +// eslint-disable-next-line no-restricted-globals +self.onmessage = (event: MessageEvent) => { + const { mangasToCheck, mangas } = event.data; + + postMessage(findDuplicatesByTitleAndAlternativeTitles(mangasToCheck, mangas)); +}; diff --git a/src/modules/library/workers/LibraryDuplicatesWorker.ts b/src/modules/library/workers/LibraryDuplicatesWorker.ts index d7a22bd9..bd97c40f 100644 --- a/src/modules/library/workers/LibraryDuplicatesWorker.ts +++ b/src/modules/library/workers/LibraryDuplicatesWorker.ts @@ -6,21 +6,66 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import { findDuplicatesByTitle } from '@/modules/library/util/LibraryDuplicates.util.ts'; import { - findDuplicatesByTitle, - findDuplicatesByTitleAndAlternativeTitles, -} from '@/modules/library/util/LibraryDuplicates.util.ts'; -import { LibraryDuplicatesWorkerInput } from '@/modules/library/Library.types.ts'; + LibraryDuplicatesDescriptionWorkerInput, + LibraryDuplicatesWorkerInput, + TMangaDuplicate, + TMangaDuplicates, +} from '@/modules/library/Library.types.ts'; +import { Queue } from '@/lib/Queue.ts'; +import { ControlledPromise } from '@/lib/ControlledPromise.ts'; +import { enhancedCleanup } from '@/util/Strings.ts'; + +const queue = new Queue((navigator.hardwareConcurrency ?? 5) - 1); +const MANGAS_PER_CHUNK = 200; // eslint-disable-next-line no-restricted-globals self.onmessage = async (event: MessageEvent) => { - const { mangas } = event.data; - const { checkAlternativeTitles } = event.data; + const { mangas, checkAlternativeTitles } = event.data; - if (checkAlternativeTitles) { - postMessage(findDuplicatesByTitleAndAlternativeTitles(mangas)); + if (!checkAlternativeTitles) { + postMessage(findDuplicatesByTitle(mangas)); return; } - postMessage(findDuplicatesByTitle(mangas)); + const chunkPromises: Promise>[] = []; + for (let chunkStart = 0; chunkStart < mangas.length; chunkStart += MANGAS_PER_CHUNK) { + chunkPromises.push( + queue.enqueue(chunkStart.toString(), () => { + const workerPromise = new ControlledPromise>(); + + const worker = new Worker(new URL('LibraryDuplicatesDescriptionWorker.ts', import.meta.url), { + type: 'module', + }); + + worker.onmessage = (subWorkerEvent: MessageEvent>) => + workerPromise.resolve(subWorkerEvent.data); + + worker.postMessage({ + mangas, + mangasToCheck: mangas.slice(chunkStart, chunkStart + MANGAS_PER_CHUNK), + } satisfies LibraryDuplicatesDescriptionWorkerInput); + + return workerPromise.promise; + }).promise, + ); + } + + const chunkedResults = await Promise.all(chunkPromises); + const mergedResult: TMangaDuplicates = {}; + + const cleanedUpTitleToOriginalTitle: Record = {}; + chunkedResults.forEach((chunkedResult) => + Object.entries(chunkedResult).forEach(([title, duplicates]) => { + const cleanedTitle = enhancedCleanup(title); + cleanedUpTitleToOriginalTitle[cleanedTitle] ??= title; + const originalTitle = cleanedUpTitleToOriginalTitle[cleanedTitle]; + + // ignore duplicated results for a title from other chunked results + mergedResult[originalTitle] ??= duplicates; + }), + ); + + postMessage(mergedResult); };