Fix saving long metadata values

fixes #900
2026-01-27 23:19:37 +01:00
parent 64df9a9489
commit 63ebfd120e
9 changed files with 390 additions and 73 deletions
--- a/src/features/metadata/services/MetadataChunker.ts
+++ b/src/features/metadata/services/MetadataChunker.ts
@@ -0,0 +1,213 @@
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+import { GqlMetaHolder, Metadata, MetadataHolderType } from '@/features/metadata/Metadata.types.ts';
+import { convertFromGqlMeta } from '@/features/metadata/services/MetadataConverter.ts';
+import { requestManager } from '@/lib/requests/RequestManager.ts';
+import { GET_GLOBAL_METADATAS } from '@/lib/graphql/metadata/GlobalMetadataQuery.ts';
+import {
+    GetGlobalMetadatasQuery,
+    GetGlobalMetadatasQueryVariables,
+    MetaInput,
+} from '@/lib/graphql/generated/graphql.ts';
+import { doesMetadataKeyExistIn } from '@/features/metadata/Metadata.utils.ts';
+
+export class MetadataChunker {
+    static readonly MAX_METADATA_VALUE_LENGTH = 4000;
+
+    static getChunkLengthKey(fullKey: string): string {
+        return `${fullKey}_length`;
+    }
+
+    static getChunkIndexKey(fullKey: string, index: number): string {
+        return `${fullKey}_${index}`;
+    }
+
+    static isChunkedInMetadata(metadata: Metadata | undefined, fullKey: string): boolean {
+        if (!metadata) {
+            return false;
+        }
+
+        return doesMetadataKeyExistIn(metadata, this.getChunkLengthKey(fullKey));
+    }
+
+    static getExistingChunkCount(metadata: Metadata | undefined, fullKey: string): number {
+        if (!metadata) {
+            return 0;
+        }
+
+        const lengthKey = this.getChunkLengthKey(fullKey);
+        if (!doesMetadataKeyExistIn(metadata, lengthKey)) {
+            return 0;
+        }
+
+        const count = Number(metadata[lengthKey]);
+
+        return Number.isNaN(count) ? 0 : count;
+    }
+
+    static getStaleChunkKeyCount(metadata: Metadata | undefined, fullKey: string): number {
+        if (!metadata) return 0;
+
+        const scanFrom = (index: number): number =>
+            doesMetadataKeyExistIn(metadata, this.getChunkIndexKey(fullKey, index)) ? scanFrom(index + 1) : index;
+
+        return scanFrom(this.getExistingChunkCount(metadata, fullKey));
+    }
+
+    static reassembleChunkedValue(metadata: Metadata, fullKey: string): string | undefined {
+        const chunkCount = this.getExistingChunkCount(metadata, fullKey);
+
+        if (chunkCount <= 0) {
+            return undefined;
+        }
+
+        return Array(chunkCount)
+            .fill(1)
+            .reduce((acc, _, index) => {
+                if (acc === undefined) {
+                    return undefined;
+                }
+
+                const chunkKey = this.getChunkIndexKey(fullKey, index);
+
+                if (!doesMetadataKeyExistIn(metadata, chunkKey)) {
+                    return undefined;
+                }
+
+                return `${acc}${metadata[chunkKey]}`;
+            }, '');
+    }
+
+    static reassembleAllChunkedValues(metadata: Metadata | undefined): Metadata | undefined {
+        if (metadata === undefined) {
+            return undefined;
+        }
+
+        const chunkedKeys = new Set<string>();
+
+        const reassembledEntries = Object.entries(metadata)
+            .map(([key]) => {
+                if (!key.endsWith('_length')) {
+                    return null;
+                }
+
+                const baseKey = key.slice(0, -'_length'.length);
+                const count = Number(metadata[key]);
+
+                if (Number.isNaN(count) || count <= 0) {
+                    return null;
+                }
+
+                const reassembledValue = this.reassembleChunkedValue(metadata, baseKey);
+
+                if (reassembledValue === undefined) {
+                    return null;
+                }
+
+                chunkedKeys.add(key);
+                for (let i = 0; i < count; i++) {
+                    chunkedKeys.add(this.getChunkIndexKey(baseKey, i));
+                }
+
+                return [baseKey, reassembledValue];
+            })
+            .filter((entry): entry is [string, string] => entry !== null);
+
+        const nonChunkedEntries = Object.entries(metadata)
+            .map(([key, value]) => {
+                if (chunkedKeys.has(key)) {
+                    return null;
+                }
+
+                return [key, value];
+            })
+            .filter((entry): entry is [string, string] => entry !== null);
+
+        const reassembledMetadata = Object.fromEntries([...nonChunkedEntries, ...reassembledEntries]);
+
+        return reassembledMetadata;
+    }
+
+    static chunkValue(fullKey: string, value: string): MetaInput[] {
+        if (value.length <= this.MAX_METADATA_VALUE_LENGTH) {
+            return [{ key: fullKey, value }];
+        }
+
+        const entries: MetaInput[] = [];
+        let offset = 0;
+        let index = 0;
+
+        while (offset < value.length) {
+            const chunk = value.substring(offset, offset + this.MAX_METADATA_VALUE_LENGTH);
+
+            entries.push({ key: this.getChunkIndexKey(fullKey, index), value: chunk });
+
+            offset += this.MAX_METADATA_VALUE_LENGTH;
+            index += 1;
+        }
+
+        entries.push({ key: this.getChunkLengthKey(fullKey), value: `${index}` });
+
+        return entries;
+    }
+
+    static computeChunkDeletions(
+        existingMetadata: Metadata | undefined,
+        fullKey: string,
+        newChunkCount: number,
+    ): string[] {
+        if (!existingMetadata) {
+            return [];
+        }
+
+        const isNowChunked = newChunkCount > 0;
+        const hasDirectKey = doesMetadataKeyExistIn(existingMetadata, fullKey);
+        const hasLengthKey = doesMetadataKeyExistIn(existingMetadata, this.getChunkLengthKey(fullKey));
+        const staleChunkKeyCount = this.getStaleChunkKeyCount(existingMetadata, fullKey);
+
+        const keysToDelete: string[] = [];
+
+        const changedToChunkedFromUnchunked = hasDirectKey && isNowChunked;
+        if (changedToChunkedFromUnchunked) {
+            keysToDelete.push(fullKey);
+        }
+
+        const changedToUnchunkedFromChunked = hasLengthKey && !isNowChunked;
+        if (changedToUnchunkedFromChunked) {
+            keysToDelete.push(this.getChunkLengthKey(fullKey));
+        }
+
+        // Delete stale chunk index keys
+        const startIndex = isNowChunked ? newChunkCount : 0;
+        for (let i = startIndex; i < staleChunkKeyCount; i++) {
+            keysToDelete.push(this.getChunkIndexKey(fullKey, i));
+        }
+
+        return keysToDelete;
+    }
+
+    static getExistingMetadata(metadataHolder: GqlMetaHolder, holderType: MetadataHolderType): Metadata | undefined {
+        if (holderType === 'global' && metadataHolder.meta === undefined) {
+            const cached = requestManager.graphQLClient.client.readQuery<
+                GetGlobalMetadatasQuery,
+                GetGlobalMetadatasQueryVariables
+            >({
+                query: GET_GLOBAL_METADATAS,
+            });
+
+            if (!cached) {
+                return undefined;
+            }
+
+            return convertFromGqlMeta(cached.metas.nodes);
+        }
+
+        return convertFromGqlMeta(metadataHolder.meta);
+    }
+}
--- a/src/features/metadata/services/MetadataMigrations.ts
+++ b/src/features/metadata/services/MetadataMigrations.ts
@@ -25,6 +25,7 @@ import { MangaIdInfo } from '@/features/manga/Manga.types.ts';
 import { CategoryIdInfo } from '@/features/category/Category.types.ts';
 import { defaultPromiseErrorHandler } from '@/lib/DefaultPromiseErrorHandler.ts';
 import { getMetadataUpdateFunction } from '@/features/metadata/services/MetadataUpdater.ts';
+import { MetadataChunker } from '@/features/metadata/services/MetadataChunker.ts';
 import { SourceIdInfo } from '@/features/source/Source.types.ts';
 import { ChapterIdInfo } from '@/features/chapter/Chapter.types.ts';

@@ -278,7 +279,8 @@ const commitMigratedMetadata = (
    migratedMetadata: Metadata,
    useEffectFn: typeof useEffect = (fn: () => void) => fn(),
 ): void => {
-    const metadata = metadataHolder?.meta;
+    const rawMetadata = metadataHolder?.meta;
+    const metadata = MetadataChunker.reassembleAllChunkedValues(rawMetadata);

    const migrationId = Number(metadata?.[getMetadataKey('migration')] ?? 1);

@@ -294,6 +296,16 @@ const commitMigratedMetadata = (
        migratedMetadata[key],
    ]) as MetadataKeyValuePair[];

+    // Expand app keys to chunk keys for deletion (e.g., some_key -> some_key_0, some_key_1, some_key_2, ..., some_key_n, some_key_length
+    const keysToDelete = metadataKeysToDelete.flatMap((fullKey) => {
+        if (rawMetadata && MetadataChunker.isChunkedInMetadata(rawMetadata, fullKey)) {
+            const count = MetadataChunker.getExistingChunkCount(rawMetadata, fullKey);
+            const chunkKeys = Array.from({ length: count }, (_, i) => MetadataChunker.getChunkIndexKey(fullKey, i));
+            return [MetadataChunker.getChunkLengthKey(fullKey), ...chunkKeys];
+        }
+        return [fullKey];
+    }) as AppMetadataKeys[];
+
    const updateMetadata = getMetadataUpdateFunction(type, metadataHolder ?? { id: -1, meta: {} });

    useEffectFn(() => {
@@ -318,7 +330,7 @@ const commitMigratedMetadata = (
            try {
                await updateMetadata({
                    update: metadataToUpdate,
-                    delete: metadataKeysToDelete,
+                    delete: keysToDelete,
                    migrate: [['migration', METADATA_MIGRATIONS.length]],
                    isMetadataKey: true,
                });
@@ -343,7 +355,8 @@ export const applyMetadataMigrations = (
        | (SourceIdInfo & MetadataHolder),
    useEffectFn: typeof useEffect = (fn: () => void) => fn(),
 ): Metadata | undefined => {
-    const meta = { ...(metadataHolder?.meta ?? {}) };
+    const rawMeta = metadataHolder?.meta ?? {};
+    const meta = MetadataChunker.reassembleAllChunkedValues(rawMeta) ?? {};

    const migrationIdKey = getMetadataKey('migration');
    const appliedMigrationId = Number.isNaN(Number(meta[migrationIdKey]))
--- a/src/features/metadata/services/MetadataReader.ts
+++ b/src/features/metadata/services/MetadataReader.ts
@@ -19,6 +19,7 @@ import { MangaIdInfo } from '@/features/manga/Manga.types.ts';

 import { CategoryIdInfo } from '@/features/category/Category.types.ts';
 import { doesMetadataKeyExistIn, getMetadataKey } from '@/features/metadata/Metadata.utils.ts';
+import { MetadataChunker } from '@/features/metadata/services/MetadataChunker.ts';
 import { applyMetadataMigrations } from '@/features/metadata/services/MetadataMigrations.ts';
 import { SourceIdInfo } from '@/features/source/Source.types.ts';
 import { ChapterIdInfo } from '@/features/chapter/Chapter.types.ts';
@@ -55,15 +56,21 @@ const getRawMetadataValueFrom = (
    key: string,
    prefixes?: string[],
 ): string | undefined => {
-    if (
-        metadata === undefined ||
-        !doesMetadataKeyExistIn(metadata, key, prefixes) ||
-        metadata[getMetadataKey(key, prefixes)] === undefined
-    ) {
+    if (metadata === undefined) {
        return undefined;
    }

-    return metadata[getMetadataKey(key, prefixes)];
+    const fullKey = getMetadataKey(key, prefixes);
+
+    if (doesMetadataKeyExistIn(metadata, fullKey) && metadata[fullKey] !== undefined) {
+        return metadata[fullKey];
+    }
+
+    if (MetadataChunker.isChunkedInMetadata(metadata, fullKey)) {
+        return MetadataChunker.reassembleChunkedValue(metadata, fullKey);
+    }
+
+    return undefined;
 };

 const getMetadataValueFrom = <Key extends AppMetadataKeys, Value extends AllowedMetadataValueTypes>(
--- a/src/features/metadata/services/MetadataUpdater.ts
+++ b/src/features/metadata/services/MetadataUpdater.ts
@@ -18,8 +18,10 @@ import {
 import { MangaIdInfo } from '@/features/manga/Manga.types.ts';
 import { getMetadataKey } from '@/features/metadata/Metadata.utils.ts';
 import { convertToGqlMeta } from '@/features/metadata/services/MetadataConverter.ts';
+import { MetadataChunker } from '@/features/metadata/services/MetadataChunker.ts';
 import { SourceIdInfo } from '@/features/source/Source.types.ts';
 import { ChapterIdInfo } from '@/features/chapter/Chapter.types.ts';
+import { MetaInput } from '@/lib/graphql/generated/graphql.ts';

 type MetadataUpdateOptions = {
    update?: MetadataKeyValuePair[];
@@ -52,19 +54,47 @@ const requestMetadataUpdate = async (
        );
    }

-    const updateMetas = keysToValues.map(([key, value]) => ({
-        key: isMetadataKey ? key : getMetadataKey(key, keyPrefixes),
-        value: `${value}`,
-    }));
-    const deleteKeys = keysToDelete.map((key) => (isMetadataKey ? key : getMetadataKey(key, keyPrefixes)));
+    const existingMetadata = MetadataChunker.getExistingMetadata(metadataHolder, holderType);
+
+    const allUpdateMetas: MetaInput[] = [];
+    const allDeleteKeys: string[] = [];
+
+    keysToValues.forEach(([key, value]) => {
+        const fullKey = isMetadataKey ? key : getMetadataKey(key, keyPrefixes);
+        const stringValue = `${value}`;
+
+        const chunkEntries = MetadataChunker.chunkValue(fullKey, stringValue);
+        allUpdateMetas.push(...chunkEntries);
+
+        const doFullCleanup = !!existingMetadata;
+        if (doFullCleanup) {
+            const newChunkCount = chunkEntries.length - 1;
+            allDeleteKeys.push(...MetadataChunker.computeChunkDeletions(existingMetadata, fullKey, newChunkCount));
+        } else {
+            const isNowChunked = chunkEntries.length > 1;
+            if (!isNowChunked) {
+                allDeleteKeys.push(MetadataChunker.getChunkLengthKey(fullKey));
+            }
+        }
+    });
+
+    keysToDelete.forEach((key) => {
+        const fullKey = isMetadataKey ? key : getMetadataKey(key, keyPrefixes);
+        allDeleteKeys.push(fullKey);
+
+        allDeleteKeys.push(...MetadataChunker.computeChunkDeletions(existingMetadata, fullKey, 0));
+    });
+
+    const uniqueDeleteKeys = [...new Set(allDeleteKeys)];
+
    const migrateMetas = keysToMigrate.map(([key, value]) => ({
        key: getMetadataKey(key, keyPrefixes),
        value: `${value}`,
    }));

    const updateMetaInput = {
-        updateInput: { metas: updateMetas },
-        deleteInput: { keys: deleteKeys },
+        updateInput: { metas: allUpdateMetas },
+        deleteInput: { keys: uniqueDeleteKeys },
        migrateInput: { metas: migrateMetas },
    };

--- a/src/features/metadata/services/MetadataValueCache.ts
+++ b/src/features/metadata/services/MetadataValueCache.ts
@@ -17,6 +17,14 @@ export class MetadataValueCache {
        return `${type}::${holderId ?? ''}::${key}`;
    }

+    static getCachedValue<T>(
+        type: MetadataHolderType,
+        holderId: string | number | undefined,
+        key: string,
+    ): T | undefined {
+        return this.convertedValueByKey.get(this.getCacheKey(type, holderId, key)) as T | undefined;
+    }
+
    static getStableValue<T>(
        type: MetadataHolderType,
        holderId: string | number | undefined,
@@ -26,7 +34,7 @@ export class MetadataValueCache {
    ): T {
        const cacheKey = this.getCacheKey(type, holderId, key);
        const cachedRawValue = this.rawValueByKey.get(cacheKey);
-        const cachedConvertedValue = this.convertedValueByKey.get(cacheKey);
+        const cachedConvertedValue = this.getCachedValue<T>(type, holderId, key);

        if (cachedRawValue !== undefined && rawValue === cachedRawValue) {
            return cachedConvertedValue as T;