[SKIP CI][Tooling] Add script to create translation changelog

.gitignore

@@ -6,6 +6,6 @@ node_modules/
 
 build/*
 
-tools/scripts/github_token.json
+tools/scripts/tokens.json
 
 src/lib/graphql/schema.json

package.json

@@ -12,6 +12,7 @@
     "build-zip": "cd build && rev=$(git rev-list HEAD --count) && echo r$rev > revision && zip -9 -r ../buildZip/Suwayomi-WebUI-r$rev *",
     "lint": "eslint src --ext .ts,.tsx,.js,.jsx --max-warnings=0 --cache",
     "createChangelog": "ts-node -T tools/scripts/createReleaseChangelog.ts",
+    "createTranslationChangelog": "ts-node -T -r tsconfig-paths/register tools/scripts/createTranslationChangelog.ts",
     "updateDeps": "ts-node -T tools/scripts/updateDependencies.ts",
     "gql:codegen-base": "graphql-codegen --config gql_codegen.ts",
     "gql:codegen-formatter": "ts-node -T tools/scripts/codegenFormatter.ts",
@@ -95,6 +96,7 @@
     "syncyarnlock": "1.0.19",
     "terser": "5.31.0",
     "ts-node": "10.9.2",
+    "tsconfig-paths": "4.2.0",
     "typescript": "5.4.5",
     "vite": "5.2.11",
     "vite-tsconfig-paths": "4.3.2",

tools/scripts/createReleaseChangelog.ts

@@ -7,7 +7,7 @@
  */
 
 import yargs from 'yargs';
-import githubToken from './github_token.json';
+import tokens from './tokens.json';
 
 const { sha } = yargs
     .options({
@@ -130,7 +130,7 @@ const fetchCommits = async (
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
-                Authorization: `Bearer ${githubToken.token}`,
+                Authorization: `Bearer ${tokens.githubToken}`,
             },
             body: JSON.stringify({ query, variables }),
         })

tools/scripts/createTranslationChangelog.ts

@@ -0,0 +1,374 @@
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+import fs from 'fs';
+import * as path from 'path';
+import * as readline from 'readline';
+import yargs from 'yargs';
+import { ControlledPromise } from '@/lib/ControlledPromise.ts';
+import tokens from './tokens.json';
+
+interface WeblateChangeResult {
+    translation: string;
+    action: number;
+    action_name: string;
+    user: string;
+}
+
+interface WeblateChangePayload {
+    next: string;
+    results: WeblateChangeResult[];
+}
+
+interface WeblateUserPayload {
+    full_name: string;
+}
+
+interface WeblateLanguagePayload {
+    language: {
+        name: string;
+    };
+}
+
+type UsernameByUserUrl = Record<string, string>;
+type LanguageNameByTranslationUrl = Record<string, string>;
+type UserUrlsByTranslationUrl = Record<string, string[]>;
+type ActionByTranslationUrlByUserUrl = Record<
+    string,
+    Record<string, { count: number; actions: Record<number, number> }>
+>;
+
+interface Contributor {
+    username: string;
+    contributionCount: number;
+}
+type ContributionsByLanguage = Record<string, Contributor[]>;
+
+enum WeblateChangeActions {
+    RESOURCE_UPDATED = 0,
+    TRANSLATION_COMPLETED = 1,
+    TRANSLATION_CHANGED = 2,
+    COMMENT_ADDED = 3,
+    SUGGESTION_ADDED = 4,
+    TRANSLATION_ADDED = 5,
+    SUGGESTION_ACCEPTED = 7,
+    TRANSLATION_REVERTED = 8,
+    TRANSLATION_UPLOADED = 9,
+    COMPONENT_LOCKED = 14,
+    COMPONENT_UNLOCKED = 15,
+    CHANGES_COMMITTED = 17,
+    CHANGES_PUSHED = 18,
+    REPOSITORY_RESET = 19,
+    REPOSITORY_MERGED = 20,
+    REPOSITORY_REBASED = 21,
+    REPOSITORY_MERGE_FAILED = 22,
+    REPOSITORY_REBASE_FAILED = 23,
+    PARSING_FAILED = 24,
+    TRANSLATION_REMOVED = 25,
+    SUGGESTION_REMOVED = 26,
+    MARKED_FOR_EDIT = 37,
+    COMPONENT_RENAMED = 42,
+    CONTRIBUTOR_JOINED = 45,
+    LANGUAGE_ADDED = 48,
+    COMPONENT_CREATED = 51,
+    ADD_ON_CONFIGURATION_CHANGED = 61,
+}
+
+const creditRelevantActions = [
+    WeblateChangeActions.TRANSLATION_CHANGED,
+    WeblateChangeActions.TRANSLATION_ADDED,
+    WeblateChangeActions.TRANSLATION_REVERTED,
+    WeblateChangeActions.TRANSLATION_UPLOADED,
+    WeblateChangeActions.TRANSLATION_REMOVED,
+    WeblateChangeActions.MARKED_FOR_EDIT,
+];
+
+const { afterDate, beforeDate } = yargs
+    .options({
+        afterDate: {
+            description: 'IS0-8601 formatted date (included) (e.g. 2024-05-01)',
+            type: 'string',
+            demandOption: true,
+        },
+        beforeDate: {
+            description: 'IS0-8601 formatted date (excluded) (e.g. 2024-05-11)',
+            type: 'string',
+            demandOption: true,
+        },
+    })
+    .parseSync();
+
+if (!afterDate || !beforeDate) {
+    throw new Error('The timespan (afterDate and beforeDate) has to be passed!');
+}
+
+const dateRegex = /[0-9]{4}-[0-9]{2}-[0-9]{2}/g;
+
+if (!afterDate.match(dateRegex) || !beforeDate.match(dateRegex)) {
+    throw new Error(
+        `The passed timestamps are not properly formatted. They have to match "${dateRegex}" (e.g. 2024-05-11)`,
+    );
+}
+
+const timestampAfter = `${afterDate}T00:00:00.000Z`;
+const timestampBefore = `${beforeDate}T00:00:00.000Z`;
+
+const fetchData = async <T = any>(url: string, authToken: string): Promise<T> => {
+    const response = await fetch(url, {
+        method: 'GET',
+        headers: {
+            Authorization: `Token ${authToken}`,
+            Accept: 'application/json',
+        },
+    });
+
+    if (!response.ok) {
+        throw new Error(`Weblate request failed with status ${response.status} - ${response.statusText}`);
+    }
+
+    return response.json();
+};
+
+const fetchWeblateChanges = async (
+    url: string,
+    authToken: string,
+    weblateChangeResults: WeblateChangeResult[] = [],
+): Promise<WeblateChangeResult[]> => {
+    const weblateChangePage = await fetchData<WeblateChangePayload>(url, authToken);
+
+    if (!weblateChangePage.next) {
+        return [...weblateChangePage.results, ...weblateChangeResults];
+    }
+
+    return fetchWeblateChanges(weblateChangePage.next, authToken, [
+        ...weblateChangePage.results,
+        ...weblateChangeResults,
+    ]);
+};
+
+const getUsername = async (url: string, authToken: string): Promise<string> => {
+    const userPayload = await fetchData<WeblateUserPayload>(url, authToken);
+    return userPayload.full_name;
+};
+
+const getLanguageName = async (url: string, authToken: string): Promise<string> => {
+    const languagePayload = await fetchData<WeblateLanguagePayload>(url, authToken);
+    return languagePayload.language.name;
+};
+
+const extractContributionInfoFromChanges = (
+    changes: WeblateChangeResult[],
+): {
+    userUrlsByTranslationUrl: UserUrlsByTranslationUrl;
+    actionsByTranslationUrlByUserUrl: ActionByTranslationUrlByUserUrl;
+    actions: Record<number, string>;
+} => {
+    const userUrlsByTranslationUrl: UserUrlsByTranslationUrl = {};
+    const actionsByTranslationUrlByUserUrl: ActionByTranslationUrlByUserUrl = {};
+    const actions: Record<number, string> = {};
+
+    changes.forEach((change) => {
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        const { translation: translationUrl, action, action_name, user: userUrl } = change;
+        if (userUrl === null) {
+            return;
+        }
+
+        userUrlsByTranslationUrl[translationUrl] = [
+            ...new Set([...(userUrlsByTranslationUrl[translationUrl] ?? []), userUrl]),
+        ];
+        actions[action] = action_name;
+        actionsByTranslationUrlByUserUrl[userUrl] = {
+            ...actionsByTranslationUrlByUserUrl[userUrl],
+            [translationUrl]: {
+                count: (actionsByTranslationUrlByUserUrl[userUrl]?.[translationUrl]?.count ?? 0) + 1,
+                actions: {
+                    ...actionsByTranslationUrlByUserUrl[userUrl]?.[translationUrl]?.actions,
+                    [action]: (actionsByTranslationUrlByUserUrl[userUrl]?.[translationUrl]?.actions[action] ?? 0) + 1,
+                },
+            },
+        };
+    });
+
+    return { userUrlsByTranslationUrl, actionsByTranslationUrlByUserUrl, actions };
+};
+
+const getUsernameByUserUrlMap = async (
+    userUrlsByTranslationUrl: UserUrlsByTranslationUrl,
+    authToken: string,
+): Promise<Record<string, string>> =>
+    Object.fromEntries(
+        (await Promise.all(
+            Object.values(userUrlsByTranslationUrl)
+                .flat()
+                .map(async (userUrl) => [userUrl, await getUsername(userUrl, authToken)]),
+        )) satisfies [string, string][],
+    );
+
+const getLanguageNameByTranslationUrl = async (
+    userUrlsByTranslationUrl: UserUrlsByTranslationUrl,
+    authToken: string,
+): Promise<LanguageNameByTranslationUrl> =>
+    Object.fromEntries(
+        (await Promise.all(
+            Object.keys(userUrlsByTranslationUrl)
+                .flat()
+                .map(async (translationUrl) => [translationUrl, await getLanguageName(translationUrl, authToken)]),
+        )) satisfies [string, string][],
+    );
+
+const getUserContributionByLanguage = (
+    userUrlsByTranslationUrl: UserUrlsByTranslationUrl,
+    languageNameByTranslationUrl: LanguageNameByTranslationUrl,
+    usernameByUserUrl: UsernameByUserUrl,
+    actionsByTranslationUrlByUserUrl: ActionByTranslationUrlByUserUrl,
+) =>
+    Object.fromEntries(
+        Object.entries(userUrlsByTranslationUrl).map(([translationUrl, userUrls]) => {
+            const languageName = languageNameByTranslationUrl[translationUrl];
+            const contributors = userUrls
+                .map((userUrl) => {
+                    const username = usernameByUserUrl[userUrl];
+                    const contributionCount = actionsByTranslationUrlByUserUrl[userUrl][translationUrl].count;
+
+                    return { username, contributionCount };
+                })
+                .sort((userA, userB) => userB.contributionCount - userA.contributionCount);
+
+            return [languageName, contributors];
+        }),
+    );
+
+const getContributorsForRange = async (url: string, authToken: string): Promise<ContributionsByLanguage> => {
+    const weblateChanges = await fetchWeblateChanges(url, authToken);
+
+    const { userUrlsByTranslationUrl, actionsByTranslationUrlByUserUrl } =
+        extractContributionInfoFromChanges(weblateChanges);
+
+    const usernameByUserUrl = await getUsernameByUserUrlMap(userUrlsByTranslationUrl, authToken);
+    const languageNameByTranslationUrl = await getLanguageNameByTranslationUrl(userUrlsByTranslationUrl, authToken);
+
+    return getUserContributionByLanguage(
+        userUrlsByTranslationUrl,
+        languageNameByTranslationUrl,
+        usernameByUserUrl,
+        actionsByTranslationUrlByUserUrl,
+    );
+};
+
+const getLanguageLinesFromChangelog = (): Promise<string[]> => {
+    const fileReadPromise = new ControlledPromise<string[]>();
+
+    const changelogFile = path.resolve(__dirname, '../../CHANGELOG.md');
+    const fileStream = fs.createReadStream(changelogFile); // Replace 'input.txt' with your file path
+    const fileLineReader = readline.createInterface({
+        input: fileStream,
+        crlfDelay: Infinity, // Recognize all line breaks (CR LF) as a single line break
+    });
+
+    const translationRelevantLines: string[] = [];
+    let isTranslationSection = false;
+    fileLineReader.on('line', (line) => {
+        if (line === '## Translations') {
+            isTranslationSection = true;
+            return;
+        }
+
+        if (isTranslationSection && line === '## Full Changelog') {
+            isTranslationSection = false;
+            return;
+        }
+
+        if (!isTranslationSection) {
+            return;
+        }
+
+        if (!line.startsWith('-')) {
+            return;
+        }
+
+        translationRelevantLines.push(line);
+    });
+
+    fileLineReader.on('close', () => {
+        fileReadPromise.resolve(translationRelevantLines);
+    });
+
+    fileLineReader.on('error', (error) => {
+        fileReadPromise.reject(error);
+    });
+
+    return fileReadPromise.promise;
+};
+
+const getKnownContributorsByLanguage = async (): Promise<Record<string, string[]>> => {
+    const changelogLanguageLines = await getLanguageLinesFromChangelog();
+
+    const contributorByLanguage: Record<string, string[]> = {};
+    changelogLanguageLines.forEach((languageLine) => {
+        const languageChangeRegex = /^- (.*) \(by (.*)\)$/g;
+        const languageChangeRegexMatch = [...languageLine.matchAll(languageChangeRegex)][0];
+
+        const language = languageChangeRegexMatch[1];
+        const contributors = languageChangeRegexMatch[2].split(', ');
+
+        const knownContributors = contributorByLanguage[language] ?? [];
+        contributorByLanguage[language] = [...new Set([...knownContributors, ...contributors])];
+    });
+
+    return contributorByLanguage;
+};
+
+const createTranslationChangelog = async (): Promise<void> => {
+    const url = `https://hosted.weblate.org/api/components/suwayomi/suwayomi-webui/changes/?timestamp_after=${timestampAfter}&timestamp_before=${timestampBefore}&${creditRelevantActions.map((action) => `action=${action}`).join('&')}`;
+
+    const contributorsForRange = await getContributorsForRange(url, tokens.weblateToken);
+    const knownContributorsByLanguage = await getKnownContributorsByLanguage();
+
+    const contributionInfo: { language: string; contributors: string[] }[] = Object.entries(contributorsForRange).map(
+        ([language, languageContributors]) => {
+            const validContributors = languageContributors.filter(
+                ({ username, contributionCount }) =>
+                    knownContributorsByLanguage[language]?.includes(username) || contributionCount >= 10,
+            );
+            const contributors = validContributors.length ? validContributors : [languageContributors[0]];
+
+            return {
+                language,
+                contributors: contributors.map(({ username }) => username),
+            };
+        },
+    );
+
+    const languageByNewState = Object.groupBy(
+        contributionInfo,
+        ({ language }) => `${knownContributorsByLanguage[language] == null}`,
+    );
+
+    const newLanguages = languageByNewState.true ?? [];
+    const updatedLanguages = languageByNewState.false ?? [];
+
+    console.log(
+        '## Translations\n' +
+            'Feel free to translate the project on [Weblate](https://hosted.weblate.org/projects/suwayomi/suwayomi-webui/)\n' +
+            '\n' +
+            'Thank you for your contribution to the translation of the project.\n',
+    );
+
+    console.log('### Added');
+    newLanguages.forEach(({ language, contributors }) => {
+        console.log(`- ${language} (by ${contributors.join(', ')})`);
+    });
+    console.log('');
+    console.log('### Updated');
+    updatedLanguages.forEach(({ language, contributors }) => {
+        console.log(`- ${language} (by ${contributors.join(', ')})`);
+    });
+};
+
+createTranslationChangelog();

tools/scripts/github_token.template.json

@@ -1,3 +0,0 @@
-{
-  "token": ""
-}

tools/scripts/token.template.json

@@ -0,0 +1,4 @@
+{
+  "githubToken": "",
+  "weblateToken": ""
+}

yarn.lock

@@ -5256,7 +5256,7 @@ json5@^1.0.2:
   dependencies:
     minimist "^1.2.0"
 
-json5@^2.2.3:
+json5@^2.2.2, json5@^2.2.3:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
   integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
@@ -6850,6 +6850,15 @@ tsconfck@^3.0.3:
   resolved "https://registry.yarnpkg.com/tsconfck/-/tsconfck-3.0.3.tgz#d9bda0e87d05b1c360e996c9050473c7e6f8084f"
   integrity sha512-4t0noZX9t6GcPTfBAbIbbIU4pfpCwh0ueq3S4O/5qXI1VwK1outmxhe9dOiEWqMz3MW2LKgDTpqWV+37IWuVbA==
 
+tsconfig-paths@4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz#ef78e19039133446d244beac0fd6a1632e2d107c"
+  integrity sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==
+  dependencies:
+    json5 "^2.2.2"
+    minimist "^1.2.6"
+    strip-bom "^3.0.0"
+
 tsconfig-paths@^3.15.0:
   version "3.15.0"
   resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz#5299ec605e55b1abb23ec939ef15edaf483070d4"