mass search (#157)

* mass search currently cripples server for like 2 mins on use * works its slow, really slow but it don't break the server anymore * concurrency using p-queue * remove package.lock + search auto select * sorting like yomi * fix yarn.lock
2022-03-12 04:46:55 +00:00
parent 841b945ca5
commit 042172a072
7 changed files with 316 additions and 12 deletions
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -40,6 +40,7 @@ import Sources from 'screens/Sources';
 import Extensions from 'screens/Extensions';
 import NavBarContextProvider from 'components/navbar/NavBarContextProvider';
 import LibraryOptionsContextProvider from 'components/library/LibraryOptionsProvider';
+import SearchAll from 'screens/SearchAll';

 declare module '@mui/styles/defaultTheme' {
    // eslint-disable-next-line @typescript-eslint/no-empty-interface
@@ -139,6 +140,9 @@ export default function App() {
                                        <Route path="/sources/:sourceId/configure/">
                                            <SourceConfigure />
                                        </Route>
+                                        <Route path="/sources/all/search/">
+                                            <SearchAll />
+                                        </Route>
                                        <Route path="/downloads">
                                            <DownloadQueue />
                                        </Route>
--- a/src/components/MangaGrid.tsx
+++ b/src/components/MangaGrid.tsx
@@ -9,6 +9,8 @@ import React, { useEffect, useRef } from 'react';
 import Grid from '@mui/material/Grid';
 import EmptyView from 'components/util/EmptyView';
 import LoadingPlaceholder from 'components/util/LoadingPlaceholder';
+import { Typography } from '@mui/material';
+import { Box } from '@mui/system';
 import MangaCard from './MangaCard';

 export interface IMangaGridProps{
@@ -20,12 +22,14 @@ export interface IMangaGridProps{
    lastPageNum: number
    setLastPageNum: (lastPageNum: number) => void
    gridLayout?: number | undefined
+    horisontal?: boolean | undefined
+    noFaces?: boolean | undefined
 }

 export default function MangaGrid(props: IMangaGridProps) {
    const {
        mangas, isLoading, message, messageExtra,
-        hasNextPage, lastPageNum, setLastPageNum, gridLayout,
+        hasNextPage, lastPageNum, setLastPageNum, gridLayout, horisontal, noFaces,
    } = props;
    let mapped;
    const lastManga = useRef<HTMLDivElement>(null);
@@ -51,7 +55,18 @@ export default function MangaGrid(props: IMangaGridProps) {
                <LoadingPlaceholder />
            );
        } else {
-            mapped = (
+            mapped = noFaces ? (
+                <Box
+                    sx={{
+                        margin: 'auto',
+                    }}
+                >
+                    <Typography variant="h5">
+                        {message}
+                    </Typography>
+                    {messageExtra}
+                </Box>
+            ) : (
                <EmptyView message={message!} messageExtra={messageExtra} />
            );
        }
@@ -78,7 +93,22 @@ export default function MangaGrid(props: IMangaGridProps) {
    }

    return (
-        <Grid container spacing={1} style={{ margin: 0, width: '100%', padding: '5px' }}>
+        <Grid
+            container
+            spacing={1}
+            style={horisontal ? {
+                margin: 0,
+                width: '100%',
+                padding: '5px',
+                overflowX: 'scroll',
+                display: '-webkit-inline-box',
+                flexWrap: 'nowrap',
+            } : {
+                margin: 0,
+                width: '100%',
+                padding: '5px',
+            }}
+        >
            {mapped}
        </Grid>
    );
--- a/src/components/util/AppbarSearch.tsx
+++ b/src/components/util/AppbarSearch.tsx
@@ -12,7 +12,16 @@ import { IconButton, Input } from '@mui/material';
 import CancelIcon from '@mui/icons-material/Cancel';
 import { useQueryParam, StringParam } from 'use-query-params';

-export default function AppbarSearch() {
+interface IProps {
+    autoOpen?: boolean
+}
+
+const defaultProps = {
+    autoOpen: false,
+};
+
+const AppbarSearch: React.FunctionComponent<IProps> = (props) => {
+    const { autoOpen } = props;
    const [query, setQuery] = useQueryParam('query', StringParam);
    const [searchOpen, setSearchOpen] = useState(!!query);
    const inputRef = React.useRef<HTMLInputElement>();
@@ -40,6 +49,12 @@ export default function AppbarSearch() {
        }
    };

+    useEffect(() => {
+        if (autoOpen) {
+            openSearch();
+        }
+    }, []);
+
    useEffect(() => {
        window.addEventListener('keydown', handleSearchShortcut);

@@ -72,4 +87,8 @@ export default function AppbarSearch() {
                )}
        </>
    );
-}
+};
+
+AppbarSearch.defaultProps = defaultProps;
+
+export default AppbarSearch;
--- a/src/screens/SearchAll.tsx
+++ b/src/screens/SearchAll.tsx
@@ -0,0 +1,224 @@
+/*
+* Copyright (C) Contributors to the Suwayomi project
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at https://mozilla.org/MPL/2.0/.
+*/
+
+import { Card } from '@mui/material';
+import { useHistory } from 'react-router-dom';
+import NavbarContext from 'components/context/NavbarContext';
+import MangaGrid from 'components/MangaGrid';
+import LangSelect from 'components/navbar/action/LangSelect';
+import AppbarSearch from 'components/util/AppbarSearch';
+import React, { useContext, useEffect, useState } from 'react';
+import { useQueryParam, StringParam } from 'use-query-params';
+import client from 'util/client';
+import {
+    langCodeToName, langSortCmp, sourceDefualtLangs, sourceForcedDefaultLangs,
+} from 'util/language';
+import useLocalStorage from 'util/useLocalStorage';
+import PQueue from 'p-queue/dist/index';
+
+function sourceToLangList(sources: ISource[]) {
+    const result: string[] = [];
+
+    sources.forEach((source) => {
+        if (result.indexOf(source.lang) === -1) { result.push(source.lang); }
+    });
+
+    result.sort(langSortCmp);
+    return result;
+}
+
+export default function SearchAll() {
+    const [query] = useQueryParam('query', StringParam);
+    const { setTitle, setAction } = useContext(NavbarContext);
+    const [triggerUpdate, setTriggerUpdate] = useState<number>(2);
+    const [mangas, setMangas] = useState<any>({});
+
+    const [shownLangs, setShownLangs] = useLocalStorage<string[]>('shownSourceLangs', sourceDefualtLangs());
+    const [showNsfw] = useLocalStorage<boolean>('showNsfw', true);
+
+    const [sources, setSources] = useState<ISource[]>([]);
+    const [fetched, setFetched] = useState<any>({});
+    const [FetchedSources, setFetchedSources] = useState<any>({});
+
+    const [lastPageNum, setLastPageNum] = useState<number>(1);
+
+    const [ResetUI, setResetUI] = useState<number>(0);
+
+    const limit = new PQueue({ concurrency: 5 });
+
+    useEffect(() => {
+        setTitle('Global Search');
+        setAction(
+            <>
+                <AppbarSearch />
+            </>
+            ,
+        );
+    }, []);
+
+    useEffect(() => {
+        client.get('/api/v1/source/list')
+            .then((response) => response.data)
+            .then((data) => {
+                setSources(data.sort((a: { displayName: string; }, b: { displayName: string; }) => {
+                    if (a.displayName < b.displayName) { return -1; }
+                    if (a.displayName > b.displayName) { return 1; }
+                    return 0;
+                })); setFetchedSources(true);
+            });
+    }, []);
+
+    async function doIT(elem: any[]) {
+        elem.map((ele) => limit.add(async () => {
+            const response = await client.get(`/api/v1/source/${ele.id}/search?searchTerm=${query || ''}&pageNum=1`);
+            const data = await response.data;
+            const tmp = mangas;
+            tmp[ele.id] = data.mangaList;
+            setMangas(tmp);
+            const tmp2 = fetched;
+            tmp2[ele.id] = true;
+            setFetched(tmp2);
+            setResetUI(1);
+        }));
+    }
+
+    useEffect(() => {
+        if (triggerUpdate === 2) {
+            return;
+        }
+        if (triggerUpdate === 0) {
+            setTriggerUpdate(1);
+            return;
+        }
+        setFetched({});
+        setMangas({});
+        // eslint-disable-next-line max-len
+        doIT(sources.filter(({ lang }) => shownLangs.indexOf(lang) !== -1).filter((source) => showNsfw || !source.isNsfw));
+    }, [triggerUpdate]);
+
+    useEffect(() => {
+        if (ResetUI === 1) {
+            setResetUI(0);
+        }
+    }, [ResetUI]);
+
+    useEffect(() => {
+        if (query && FetchedSources) {
+            const delayDebounceFn = setTimeout(() => {
+                setTriggerUpdate(0);
+            }, 1000);
+            return () => clearTimeout(delayDebounceFn);
+        }
+        return () => {};
+    }, [query, shownLangs, sources]);
+
+    useEffect(() => {
+        // make sure all of forcedDefaultLangs() exists in shownLangs
+        sourceForcedDefaultLangs().forEach((forcedLang) => {
+            let hasLang = false;
+            shownLangs.forEach((lang) => {
+                if (lang === forcedLang) hasLang = true;
+            });
+            if (!hasLang) {
+                setShownLangs((shownLangsCopy) => {
+                    shownLangsCopy.push(forcedLang);
+                    return shownLangsCopy;
+                });
+            }
+        });
+    }, []);
+
+    useEffect(() => {
+        setTitle('Sources');
+        setAction(
+            <>
+                <AppbarSearch
+                    autoOpen
+                />
+                <LangSelect
+                    shownLangs={shownLangs}
+                    setShownLangs={setShownLangs}
+                    allLangs={sourceToLangList(sources)}
+                    forcedLangs={sourceForcedDefaultLangs()}
+                />
+            </>,
+        );
+    }, [shownLangs, sources]);
+
+    const history = useHistory();
+
+    const redirectTo = (e: any, to: string) => {
+        history.push(to);
+
+        // prevent parent tags from getting the event
+        e.stopPropagation();
+    };
+    if (query) {
+        return (
+            <>
+                {/* eslint-disable-next-line max-len */}
+                {sources.filter(({ lang }) => shownLangs.indexOf(lang) !== -1).filter((source) => showNsfw || !source.isNsfw).sort((a, b) => {
+                    const af = fetched[a.id];
+                    const bf = fetched[b.id];
+                    if (af && !bf) { return -1; }
+                    if (!af && bf) { return 1; }
+                    if (!af && !bf) { return 0; }
+
+                    const al = mangas[a.id].length === 0;
+                    const bl = mangas[b.id].length === 0;
+                    if (al && !bl) { return 1; }
+                    if (bl && !al) { return -1; }
+                    return 0;
+                }).map(({ lang, id, displayName }) => (
+                    (
+                        <>
+                            <Card
+                                sx={{
+                                    margin: '10px',
+                                    '&:hover': {
+                                        backgroundColor: 'action.hover',
+                                        transition: 'background-color 100ms cubic-bezier(0.4, 0, 0.2, 1) 0ms',
+                                    },
+                                    '&:active': {
+                                        backgroundColor: 'action.selected',
+                                        transition: 'background-color 100ms cubic-bezier(0.4, 0, 0.2, 1) 0ms',
+                                    },
+                                }}
+                                onClick={(e) => redirectTo(e, `/sources/${id}/popular/?R&query=one`)}
+                            >
+                                <h1
+                                    key={lang}
+                                    style={{ margin: '25px 0px 0px 25px' }}
+                                >
+                                    {displayName}
+                                </h1>
+                                <p
+                                    style={{ margin: '0px 0px 25px 25px' }}
+                                >
+                                    {langCodeToName(lang)}
+                                </p>
+                            </Card>
+                            <MangaGrid
+                                mangas={mangas[id] || []}
+                                isLoading={!fetched[id]}
+                                hasNextPage={false}
+                                lastPageNum={lastPageNum}
+                                setLastPageNum={setLastPageNum}
+                                horisontal
+                                noFaces
+                                message={fetched[id] ? 'No manga was found!' : undefined}
+                            />
+                        </>
+                    )
+                ))}
+
+            </>
+        );
+    }
+    return (<></>);
+}
--- a/src/screens/Sources.tsx
+++ b/src/screens/Sources.tsx
@@ -15,6 +15,9 @@ import {
 } from 'util/language';
 import useLocalStorage from 'util/useLocalStorage';
 import LoadingPlaceholder from 'components/util/LoadingPlaceholder';
+import { IconButton } from '@mui/material';
+import TravelExploreIcon from '@mui/icons-material/TravelExplore';
+import { useHistory } from 'react-router-dom';

 function sourceToLangList(sources: ISource[]) {
    const result: string[] = [];
@@ -46,6 +49,8 @@ export default function Sources() {
    const [sources, setSources] = useState<ISource[]>([]);
    const [fetched, setFetched] = useState<boolean>(false);

+    const history = useHistory();
+
    useEffect(() => {
        // make sure all of forcedDefaultLangs() exists in shownLangs
        sourceForcedDefaultLangs().forEach((forcedLang) => {
@@ -65,12 +70,20 @@ export default function Sources() {
    useEffect(() => {
        setTitle('Sources');
        setAction(
-            <LangSelect
-                shownLangs={shownLangs}
-                setShownLangs={setShownLangs}
-                allLangs={sourceToLangList(sources)}
-                forcedLangs={sourceForcedDefaultLangs()}
-            />,
+            <>
+                <IconButton
+                    onClick={() => history.push('/sources/all/search/')}
+                    size="large"
+                >
+                    <TravelExploreIcon />
+                </IconButton>
+                <LangSelect
+                    shownLangs={shownLangs}
+                    setShownLangs={setShownLangs}
+                    allLangs={sourceToLangList(sources)}
+                    forcedLangs={sourceForcedDefaultLangs()}
+                />
+            </>,
        );
    }, [shownLangs, sources]);