Improve library duplicates detection performance

Parallelize by chunking the manga list and using sub workers

src/modules/library/Library.types.ts

@@ -55,3 +55,8 @@ export type LibraryDuplicatesWorkerInput<Manga extends TMangaDuplicate = TMangaD
     mangas: Manga[];
     checkAlternativeTitles: boolean;
 };
+
+export type LibraryDuplicatesDescriptionWorkerInput<Manga extends TMangaDuplicate = TMangaDuplicate> = {
+    mangasToCheck: Manga[];
+    mangas: Manga[];
+};

src/modules/library/util/LibraryDuplicates.util.ts

@@ -60,12 +60,13 @@ const findDuplicatesByTitleAndAlternativeTitlesSingleManga = <Manga extends TMan
 };
 
 export const findDuplicatesByTitleAndAlternativeTitles = <Manga extends TMangaDuplicate>(
-    mangas: Manga[],
+    mangasToCheck: Manga[],
-): Promise<TMangaDuplicates<Manga>> => {
+    mangas: Manga[] = mangasToCheck,
+): TMangaDuplicates<Manga> => {
     const titleToMangas: TMangaDuplicates<Manga> = {};
     const titleToAlternativeTitleMatches: TMangaDuplicates<Manga> = {};
 
-    mangas.forEach((mangaToCheck) => {
+    mangasToCheck.forEach((mangaToCheck) => {
         const titleToCheck = enhancedCleanup(mangaToCheck.title);
 
         titleToMangas[titleToCheck] ??= [];

src/modules/library/workers/LibraryDuplicatesDescriptionWorker.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+import { findDuplicatesByTitleAndAlternativeTitles } from '@/modules/library/util/LibraryDuplicates.util.ts';
+import { LibraryDuplicatesDescriptionWorkerInput } from '@/modules/library/Library.types.ts';
+
+// eslint-disable-next-line no-restricted-globals
+self.onmessage = (event: MessageEvent<LibraryDuplicatesDescriptionWorkerInput>) => {
+    const { mangasToCheck, mangas } = event.data;
+
+    postMessage(findDuplicatesByTitleAndAlternativeTitles(mangasToCheck, mangas));
+};

src/modules/library/workers/LibraryDuplicatesWorker.ts

@@ -6,21 +6,66 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
+import { findDuplicatesByTitle } from '@/modules/library/util/LibraryDuplicates.util.ts';
 import {
-    findDuplicatesByTitle,
+    LibraryDuplicatesDescriptionWorkerInput,
-    findDuplicatesByTitleAndAlternativeTitles,
+    LibraryDuplicatesWorkerInput,
-} from '@/modules/library/util/LibraryDuplicates.util.ts';
+    TMangaDuplicate,
-import { LibraryDuplicatesWorkerInput } from '@/modules/library/Library.types.ts';
+    TMangaDuplicates,
+} from '@/modules/library/Library.types.ts';
+import { Queue } from '@/lib/Queue.ts';
+import { ControlledPromise } from '@/lib/ControlledPromise.ts';
+import { enhancedCleanup } from '@/util/Strings.ts';
+
+const queue = new Queue((navigator.hardwareConcurrency ?? 5) - 1);
+const MANGAS_PER_CHUNK = 200;
 
 // eslint-disable-next-line no-restricted-globals
 self.onmessage = async (event: MessageEvent<LibraryDuplicatesWorkerInput>) => {
-    const { mangas } = event.data;
+    const { mangas, checkAlternativeTitles } = event.data;
-    const { checkAlternativeTitles } = event.data;
 
-    if (checkAlternativeTitles) {
+    if (!checkAlternativeTitles) {
-        postMessage(findDuplicatesByTitleAndAlternativeTitles(mangas));
+        postMessage(findDuplicatesByTitle(mangas));
         return;
     }
 
-    postMessage(findDuplicatesByTitle(mangas));
+    const chunkPromises: Promise<TMangaDuplicates<TMangaDuplicate>>[] = [];
+    for (let chunkStart = 0; chunkStart < mangas.length; chunkStart += MANGAS_PER_CHUNK) {
+        chunkPromises.push(
+            queue.enqueue(chunkStart.toString(), () => {
+                const workerPromise = new ControlledPromise<TMangaDuplicates<TMangaDuplicate>>();
+
+                const worker = new Worker(new URL('LibraryDuplicatesDescriptionWorker.ts', import.meta.url), {
+                    type: 'module',
+                });
+
+                worker.onmessage = (subWorkerEvent: MessageEvent<TMangaDuplicates<TMangaDuplicate>>) =>
+                    workerPromise.resolve(subWorkerEvent.data);
+
+                worker.postMessage({
+                    mangas,
+                    mangasToCheck: mangas.slice(chunkStart, chunkStart + MANGAS_PER_CHUNK),
+                } satisfies LibraryDuplicatesDescriptionWorkerInput);
+
+                return workerPromise.promise;
+            }).promise,
+        );
+    }
+
+    const chunkedResults = await Promise.all(chunkPromises);
+    const mergedResult: TMangaDuplicates<TMangaDuplicate> = {};
+
+    const cleanedUpTitleToOriginalTitle: Record<string, string> = {};
+    chunkedResults.forEach((chunkedResult) =>
+        Object.entries(chunkedResult).forEach(([title, duplicates]) => {
+            const cleanedTitle = enhancedCleanup(title);
+            cleanedUpTitleToOriginalTitle[cleanedTitle] ??= title;
+            const originalTitle = cleanedUpTitleToOriginalTitle[cleanedTitle];
+
+            // ignore duplicated results for a title from other chunked results
+            mergedResult[originalTitle] ??= duplicates;
+        }),
+    );
+
+    postMessage(mergedResult);
 };