mass search (#157)

* mass search

currently cripples server for like 2 mins on use

* works

its slow, really slow
but it don't break the server anymore

* concurrency using p-queue

* remove package.lock + search auto select

* sorting like yomi

* fix yarn.lock

package.json

@@ -13,6 +13,7 @@
     "@mui/system": "^5.0.6",
     "axios": "^0.21.1",
     "file-selector": "^0.2.4",
+    "p-queue": "^7.2.0",
     "query-string": "^7.0.1",
     "react": "^17.0.2",
     "react-beautiful-dnd": "^13.0.0",

src/App.tsx

@@ -40,6 +40,7 @@ import Sources from 'screens/Sources';
 import Extensions from 'screens/Extensions';
 import NavBarContextProvider from 'components/navbar/NavBarContextProvider';
 import LibraryOptionsContextProvider from 'components/library/LibraryOptionsProvider';
+import SearchAll from 'screens/SearchAll';
 
 declare module '@mui/styles/defaultTheme' {
     // eslint-disable-next-line @typescript-eslint/no-empty-interface
@@ -139,6 +140,9 @@ export default function App() {
                                         <Route path="/sources/:sourceId/configure/">
                                             <SourceConfigure />
                                         </Route>
+                                        <Route path="/sources/all/search/">
+                                            <SearchAll />
+                                        </Route>
                                         <Route path="/downloads">
                                             <DownloadQueue />
                                         </Route>

src/components/MangaGrid.tsx

@@ -9,6 +9,8 @@ import React, { useEffect, useRef } from 'react';
 import Grid from '@mui/material/Grid';
 import EmptyView from 'components/util/EmptyView';
 import LoadingPlaceholder from 'components/util/LoadingPlaceholder';
+import { Typography } from '@mui/material';
+import { Box } from '@mui/system';
 import MangaCard from './MangaCard';
 
 export interface IMangaGridProps{
@@ -20,12 +22,14 @@ export interface IMangaGridProps{
     lastPageNum: number
     setLastPageNum: (lastPageNum: number) => void
     gridLayout?: number | undefined
+    horisontal?: boolean | undefined
+    noFaces?: boolean | undefined
 }
 
 export default function MangaGrid(props: IMangaGridProps) {
     const {
         mangas, isLoading, message, messageExtra,
-        hasNextPage, lastPageNum, setLastPageNum, gridLayout,
+        hasNextPage, lastPageNum, setLastPageNum, gridLayout, horisontal, noFaces,
     } = props;
     let mapped;
     const lastManga = useRef<HTMLDivElement>(null);
@@ -51,7 +55,18 @@ export default function MangaGrid(props: IMangaGridProps) {
                 <LoadingPlaceholder />
             );
         } else {
-            mapped = (
+            mapped = noFaces ? (
+                <Box
+                    sx={{
+                        margin: 'auto',
+                    }}
+                >
+                    <Typography variant="h5">
+                        {message}
+                    </Typography>
+                    {messageExtra}
+                </Box>
+            ) : (
                 <EmptyView message={message!} messageExtra={messageExtra} />
             );
         }
@@ -78,7 +93,22 @@ export default function MangaGrid(props: IMangaGridProps) {
     }
 
     return (
-        <Grid container spacing={1} style={{ margin: 0, width: '100%', padding: '5px' }}>
+        <Grid
+            container
+            spacing={1}
+            style={horisontal ? {
+                margin: 0,
+                width: '100%',
+                padding: '5px',
+                overflowX: 'scroll',
+                display: '-webkit-inline-box',
+                flexWrap: 'nowrap',
+            } : {
+                margin: 0,
+                width: '100%',
+                padding: '5px',
+            }}
+        >
             {mapped}
         </Grid>
     );

src/components/util/AppbarSearch.tsx

@@ -12,7 +12,16 @@ import { IconButton, Input } from '@mui/material';
 import CancelIcon from '@mui/icons-material/Cancel';
 import { useQueryParam, StringParam } from 'use-query-params';
 
-export default function AppbarSearch() {
+interface IProps {
+    autoOpen?: boolean
+}
+
+const defaultProps = {
+    autoOpen: false,
+};
+
+const AppbarSearch: React.FunctionComponent<IProps> = (props) => {
+    const { autoOpen } = props;
     const [query, setQuery] = useQueryParam('query', StringParam);
     const [searchOpen, setSearchOpen] = useState(!!query);
     const inputRef = React.useRef<HTMLInputElement>();
@@ -40,6 +49,12 @@ export default function AppbarSearch() {
         }
     };
 
+    useEffect(() => {
+        if (autoOpen) {
+            openSearch();
+        }
+    }, []);
+
     useEffect(() => {
         window.addEventListener('keydown', handleSearchShortcut);
 
@@ -72,4 +87,8 @@ export default function AppbarSearch() {
                 )}
         </>
     );
-}
+};
+
+AppbarSearch.defaultProps = defaultProps;
+
+export default AppbarSearch;

src/screens/SearchAll.tsx

@@ -0,0 +1,224 @@
+/*
+* Copyright (C) Contributors to the Suwayomi project
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at https://mozilla.org/MPL/2.0/.
+*/
+
+import { Card } from '@mui/material';
+import { useHistory } from 'react-router-dom';
+import NavbarContext from 'components/context/NavbarContext';
+import MangaGrid from 'components/MangaGrid';
+import LangSelect from 'components/navbar/action/LangSelect';
+import AppbarSearch from 'components/util/AppbarSearch';
+import React, { useContext, useEffect, useState } from 'react';
+import { useQueryParam, StringParam } from 'use-query-params';
+import client from 'util/client';
+import {
+    langCodeToName, langSortCmp, sourceDefualtLangs, sourceForcedDefaultLangs,
+} from 'util/language';
+import useLocalStorage from 'util/useLocalStorage';
+import PQueue from 'p-queue/dist/index';
+
+function sourceToLangList(sources: ISource[]) {
+    const result: string[] = [];
+
+    sources.forEach((source) => {
+        if (result.indexOf(source.lang) === -1) { result.push(source.lang); }
+    });
+
+    result.sort(langSortCmp);
+    return result;
+}
+
+export default function SearchAll() {
+    const [query] = useQueryParam('query', StringParam);
+    const { setTitle, setAction } = useContext(NavbarContext);
+    const [triggerUpdate, setTriggerUpdate] = useState<number>(2);
+    const [mangas, setMangas] = useState<any>({});
+
+    const [shownLangs, setShownLangs] = useLocalStorage<string[]>('shownSourceLangs', sourceDefualtLangs());
+    const [showNsfw] = useLocalStorage<boolean>('showNsfw', true);
+
+    const [sources, setSources] = useState<ISource[]>([]);
+    const [fetched, setFetched] = useState<any>({});
+    const [FetchedSources, setFetchedSources] = useState<any>({});
+
+    const [lastPageNum, setLastPageNum] = useState<number>(1);
+
+    const [ResetUI, setResetUI] = useState<number>(0);
+
+    const limit = new PQueue({ concurrency: 5 });
+
+    useEffect(() => {
+        setTitle('Global Search');
+        setAction(
+            <>
+                <AppbarSearch />
+            </>
+            ,
+        );
+    }, []);
+
+    useEffect(() => {
+        client.get('/api/v1/source/list')
+            .then((response) => response.data)
+            .then((data) => {
+                setSources(data.sort((a: { displayName: string; }, b: { displayName: string; }) => {
+                    if (a.displayName < b.displayName) { return -1; }
+                    if (a.displayName > b.displayName) { return 1; }
+                    return 0;
+                })); setFetchedSources(true);
+            });
+    }, []);
+
+    async function doIT(elem: any[]) {
+        elem.map((ele) => limit.add(async () => {
+            const response = await client.get(`/api/v1/source/${ele.id}/search?searchTerm=${query || ''}&pageNum=1`);
+            const data = await response.data;
+            const tmp = mangas;
+            tmp[ele.id] = data.mangaList;
+            setMangas(tmp);
+            const tmp2 = fetched;
+            tmp2[ele.id] = true;
+            setFetched(tmp2);
+            setResetUI(1);
+        }));
+    }
+
+    useEffect(() => {
+        if (triggerUpdate === 2) {
+            return;
+        }
+        if (triggerUpdate === 0) {
+            setTriggerUpdate(1);
+            return;
+        }
+        setFetched({});
+        setMangas({});
+        // eslint-disable-next-line max-len
+        doIT(sources.filter(({ lang }) => shownLangs.indexOf(lang) !== -1).filter((source) => showNsfw || !source.isNsfw));
+    }, [triggerUpdate]);
+
+    useEffect(() => {
+        if (ResetUI === 1) {
+            setResetUI(0);
+        }
+    }, [ResetUI]);
+
+    useEffect(() => {
+        if (query && FetchedSources) {
+            const delayDebounceFn = setTimeout(() => {
+                setTriggerUpdate(0);
+            }, 1000);
+            return () => clearTimeout(delayDebounceFn);
+        }
+        return () => {};
+    }, [query, shownLangs, sources]);
+
+    useEffect(() => {
+        // make sure all of forcedDefaultLangs() exists in shownLangs
+        sourceForcedDefaultLangs().forEach((forcedLang) => {
+            let hasLang = false;
+            shownLangs.forEach((lang) => {
+                if (lang === forcedLang) hasLang = true;
+            });
+            if (!hasLang) {
+                setShownLangs((shownLangsCopy) => {
+                    shownLangsCopy.push(forcedLang);
+                    return shownLangsCopy;
+                });
+            }
+        });
+    }, []);
+
+    useEffect(() => {
+        setTitle('Sources');
+        setAction(
+            <>
+                <AppbarSearch
+                    autoOpen
+                />
+                <LangSelect
+                    shownLangs={shownLangs}
+                    setShownLangs={setShownLangs}
+                    allLangs={sourceToLangList(sources)}
+                    forcedLangs={sourceForcedDefaultLangs()}
+                />
+            </>,
+        );
+    }, [shownLangs, sources]);
+
+    const history = useHistory();
+
+    const redirectTo = (e: any, to: string) => {
+        history.push(to);
+
+        // prevent parent tags from getting the event
+        e.stopPropagation();
+    };
+    if (query) {
+        return (
+            <>
+                {/* eslint-disable-next-line max-len */}
+                {sources.filter(({ lang }) => shownLangs.indexOf(lang) !== -1).filter((source) => showNsfw || !source.isNsfw).sort((a, b) => {
+                    const af = fetched[a.id];
+                    const bf = fetched[b.id];
+                    if (af && !bf) { return -1; }
+                    if (!af && bf) { return 1; }
+                    if (!af && !bf) { return 0; }
+
+                    const al = mangas[a.id].length === 0;
+                    const bl = mangas[b.id].length === 0;
+                    if (al && !bl) { return 1; }
+                    if (bl && !al) { return -1; }
+                    return 0;
+                }).map(({ lang, id, displayName }) => (
+                    (
+                        <>
+                            <Card
+                                sx={{
+                                    margin: '10px',
+                                    '&:hover': {
+                                        backgroundColor: 'action.hover',
+                                        transition: 'background-color 100ms cubic-bezier(0.4, 0, 0.2, 1) 0ms',
+                                    },
+                                    '&:active': {
+                                        backgroundColor: 'action.selected',
+                                        transition: 'background-color 100ms cubic-bezier(0.4, 0, 0.2, 1) 0ms',
+                                    },
+                                }}
+                                onClick={(e) => redirectTo(e, `/sources/${id}/popular/?R&query=one`)}
+                            >
+                                <h1
+                                    key={lang}
+                                    style={{ margin: '25px 0px 0px 25px' }}
+                                >
+                                    {displayName}
+                                </h1>
+                                <p
+                                    style={{ margin: '0px 0px 25px 25px' }}
+                                >
+                                    {langCodeToName(lang)}
+                                </p>
+                            </Card>
+                            <MangaGrid
+                                mangas={mangas[id] || []}
+                                isLoading={!fetched[id]}
+                                hasNextPage={false}
+                                lastPageNum={lastPageNum}
+                                setLastPageNum={setLastPageNum}
+                                horisontal
+                                noFaces
+                                message={fetched[id] ? 'No manga was found!' : undefined}
+                            />
+                        </>
+                    )
+                ))}
+
+            </>
+        );
+    }
+    return (<></>);
+}

src/screens/Sources.tsx

@@ -15,6 +15,9 @@ import {
 } from 'util/language';
 import useLocalStorage from 'util/useLocalStorage';
 import LoadingPlaceholder from 'components/util/LoadingPlaceholder';
+import { IconButton } from '@mui/material';
+import TravelExploreIcon from '@mui/icons-material/TravelExplore';
+import { useHistory } from 'react-router-dom';
 
 function sourceToLangList(sources: ISource[]) {
     const result: string[] = [];
@@ -46,6 +49,8 @@ export default function Sources() {
     const [sources, setSources] = useState<ISource[]>([]);
     const [fetched, setFetched] = useState<boolean>(false);
 
+    const history = useHistory();
+
     useEffect(() => {
         // make sure all of forcedDefaultLangs() exists in shownLangs
         sourceForcedDefaultLangs().forEach((forcedLang) => {
@@ -65,12 +70,20 @@ export default function Sources() {
     useEffect(() => {
         setTitle('Sources');
         setAction(
-            <LangSelect
+            <>
-                shownLangs={shownLangs}
+                <IconButton
-                setShownLangs={setShownLangs}
+                    onClick={() => history.push('/sources/all/search/')}
-                allLangs={sourceToLangList(sources)}
+                    size="large"
-                forcedLangs={sourceForcedDefaultLangs()}
+                >
-            />,
+                    <TravelExploreIcon />
+                </IconButton>
+                <LangSelect
+                    shownLangs={shownLangs}
+                    setShownLangs={setShownLangs}
+                    allLangs={sourceToLangList(sources)}
+                    forcedLangs={sourceForcedDefaultLangs()}
+                />
+            </>,
         );
     }, [shownLangs, sources]);
 

yarn.lock

@@ -5174,7 +5174,7 @@ etag@~1.8.1:
   resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
   integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
 
-eventemitter3@^4.0.0:
+eventemitter3@^4.0.0, eventemitter3@^4.0.7:
   version "4.0.7"
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
   integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
@@ -8369,6 +8369,14 @@ p-map@^4.0.0:
   dependencies:
     aggregate-error "^3.0.0"
 
+p-queue@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-7.2.0.tgz#e1430e4432f09b43aa8b4913d4c2ff7fdd685479"
+  integrity sha512-Kvv7p13M46lTYLQ/PsZdaj/1Vj6u/8oiIJgyQyx4oVkOfHdd7M2EZvXigDvcsSzRwanCzQirV5bJPQFoSQt5MA==
+  dependencies:
+    eventemitter3 "^4.0.7"
+    p-timeout "^5.0.2"
+
 p-retry@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-3.0.1.tgz#316b4c8893e2c8dc1cfa891f406c4b422bebf328"
@@ -8376,6 +8384,11 @@ p-retry@^3.0.1:
   dependencies:
     retry "^0.12.0"
 
+p-timeout@^5.0.2:
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-5.0.2.tgz#d12964c4b2f988e15f72b455c2c428d82a0ec0a0"
+  integrity sha512-sEmji9Yaq+Tw+STwsGAE56hf7gMy9p0tQfJojIAamB7WHJYJKf1qlsg9jqBWG8q9VCxKPhZaP/AcXwEoBcYQhQ==
+
 p-try@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"