diff --git a/.gitignore b/.gitignore index 6268e775..ee93e851 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,6 @@ node_modules/ build/* -tools/scripts/github_token.json +tools/scripts/tokens.json src/lib/graphql/schema.json \ No newline at end of file diff --git a/package.json b/package.json index 9e15c5d6..c0c3d566 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "build-zip": "cd build && rev=$(git rev-list HEAD --count) && echo r$rev > revision && zip -9 -r ../buildZip/Suwayomi-WebUI-r$rev *", "lint": "eslint src --ext .ts,.tsx,.js,.jsx --max-warnings=0 --cache", "createChangelog": "ts-node -T tools/scripts/createReleaseChangelog.ts", + "createTranslationChangelog": "ts-node -T -r tsconfig-paths/register tools/scripts/createTranslationChangelog.ts", "updateDeps": "ts-node -T tools/scripts/updateDependencies.ts", "gql:codegen-base": "graphql-codegen --config gql_codegen.ts", "gql:codegen-formatter": "ts-node -T tools/scripts/codegenFormatter.ts", @@ -95,6 +96,7 @@ "syncyarnlock": "1.0.19", "terser": "5.31.0", "ts-node": "10.9.2", + "tsconfig-paths": "4.2.0", "typescript": "5.4.5", "vite": "5.2.11", "vite-tsconfig-paths": "4.3.2", diff --git a/tools/scripts/createReleaseChangelog.ts b/tools/scripts/createReleaseChangelog.ts index b5b952ee..37aa23b0 100644 --- a/tools/scripts/createReleaseChangelog.ts +++ b/tools/scripts/createReleaseChangelog.ts @@ -7,7 +7,7 @@ */ import yargs from 'yargs'; -import githubToken from './github_token.json'; +import tokens from './tokens.json'; const { sha } = yargs .options({ @@ -130,7 +130,7 @@ const fetchCommits = async ( method: 'POST', headers: { 'Content-Type': 'application/json', - Authorization: `Bearer ${githubToken.token}`, + Authorization: `Bearer ${tokens.githubToken}`, }, body: JSON.stringify({ query, variables }), }) diff --git a/tools/scripts/createTranslationChangelog.ts b/tools/scripts/createTranslationChangelog.ts new file mode 100644 index 00000000..d217f9b8 --- /dev/null +++ b/tools/scripts/createTranslationChangelog.ts @@ -0,0 +1,374 @@ +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import fs from 'fs'; +import * as path from 'path'; +import * as readline from 'readline'; +import yargs from 'yargs'; +import { ControlledPromise } from '@/lib/ControlledPromise.ts'; +import tokens from './tokens.json'; + +interface WeblateChangeResult { + translation: string; + action: number; + action_name: string; + user: string; +} + +interface WeblateChangePayload { + next: string; + results: WeblateChangeResult[]; +} + +interface WeblateUserPayload { + full_name: string; +} + +interface WeblateLanguagePayload { + language: { + name: string; + }; +} + +type UsernameByUserUrl = Record; +type LanguageNameByTranslationUrl = Record; +type UserUrlsByTranslationUrl = Record; +type ActionByTranslationUrlByUserUrl = Record< + string, + Record }> +>; + +interface Contributor { + username: string; + contributionCount: number; +} +type ContributionsByLanguage = Record; + +enum WeblateChangeActions { + RESOURCE_UPDATED = 0, + TRANSLATION_COMPLETED = 1, + TRANSLATION_CHANGED = 2, + COMMENT_ADDED = 3, + SUGGESTION_ADDED = 4, + TRANSLATION_ADDED = 5, + SUGGESTION_ACCEPTED = 7, + TRANSLATION_REVERTED = 8, + TRANSLATION_UPLOADED = 9, + COMPONENT_LOCKED = 14, + COMPONENT_UNLOCKED = 15, + CHANGES_COMMITTED = 17, + CHANGES_PUSHED = 18, + REPOSITORY_RESET = 19, + REPOSITORY_MERGED = 20, + REPOSITORY_REBASED = 21, + REPOSITORY_MERGE_FAILED = 22, + REPOSITORY_REBASE_FAILED = 23, + PARSING_FAILED = 24, + TRANSLATION_REMOVED = 25, + SUGGESTION_REMOVED = 26, + MARKED_FOR_EDIT = 37, + COMPONENT_RENAMED = 42, + CONTRIBUTOR_JOINED = 45, + LANGUAGE_ADDED = 48, + COMPONENT_CREATED = 51, + ADD_ON_CONFIGURATION_CHANGED = 61, +} + +const creditRelevantActions = [ + WeblateChangeActions.TRANSLATION_CHANGED, + WeblateChangeActions.TRANSLATION_ADDED, + WeblateChangeActions.TRANSLATION_REVERTED, + WeblateChangeActions.TRANSLATION_UPLOADED, + WeblateChangeActions.TRANSLATION_REMOVED, + WeblateChangeActions.MARKED_FOR_EDIT, +]; + +const { afterDate, beforeDate } = yargs + .options({ + afterDate: { + description: 'IS0-8601 formatted date (included) (e.g. 2024-05-01)', + type: 'string', + demandOption: true, + }, + beforeDate: { + description: 'IS0-8601 formatted date (excluded) (e.g. 2024-05-11)', + type: 'string', + demandOption: true, + }, + }) + .parseSync(); + +if (!afterDate || !beforeDate) { + throw new Error('The timespan (afterDate and beforeDate) has to be passed!'); +} + +const dateRegex = /[0-9]{4}-[0-9]{2}-[0-9]{2}/g; + +if (!afterDate.match(dateRegex) || !beforeDate.match(dateRegex)) { + throw new Error( + `The passed timestamps are not properly formatted. They have to match "${dateRegex}" (e.g. 2024-05-11)`, + ); +} + +const timestampAfter = `${afterDate}T00:00:00.000Z`; +const timestampBefore = `${beforeDate}T00:00:00.000Z`; + +const fetchData = async (url: string, authToken: string): Promise => { + const response = await fetch(url, { + method: 'GET', + headers: { + Authorization: `Token ${authToken}`, + Accept: 'application/json', + }, + }); + + if (!response.ok) { + throw new Error(`Weblate request failed with status ${response.status} - ${response.statusText}`); + } + + return response.json(); +}; + +const fetchWeblateChanges = async ( + url: string, + authToken: string, + weblateChangeResults: WeblateChangeResult[] = [], +): Promise => { + const weblateChangePage = await fetchData(url, authToken); + + if (!weblateChangePage.next) { + return [...weblateChangePage.results, ...weblateChangeResults]; + } + + return fetchWeblateChanges(weblateChangePage.next, authToken, [ + ...weblateChangePage.results, + ...weblateChangeResults, + ]); +}; + +const getUsername = async (url: string, authToken: string): Promise => { + const userPayload = await fetchData(url, authToken); + return userPayload.full_name; +}; + +const getLanguageName = async (url: string, authToken: string): Promise => { + const languagePayload = await fetchData(url, authToken); + return languagePayload.language.name; +}; + +const extractContributionInfoFromChanges = ( + changes: WeblateChangeResult[], +): { + userUrlsByTranslationUrl: UserUrlsByTranslationUrl; + actionsByTranslationUrlByUserUrl: ActionByTranslationUrlByUserUrl; + actions: Record; +} => { + const userUrlsByTranslationUrl: UserUrlsByTranslationUrl = {}; + const actionsByTranslationUrlByUserUrl: ActionByTranslationUrlByUserUrl = {}; + const actions: Record = {}; + + changes.forEach((change) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { translation: translationUrl, action, action_name, user: userUrl } = change; + if (userUrl === null) { + return; + } + + userUrlsByTranslationUrl[translationUrl] = [ + ...new Set([...(userUrlsByTranslationUrl[translationUrl] ?? []), userUrl]), + ]; + actions[action] = action_name; + actionsByTranslationUrlByUserUrl[userUrl] = { + ...actionsByTranslationUrlByUserUrl[userUrl], + [translationUrl]: { + count: (actionsByTranslationUrlByUserUrl[userUrl]?.[translationUrl]?.count ?? 0) + 1, + actions: { + ...actionsByTranslationUrlByUserUrl[userUrl]?.[translationUrl]?.actions, + [action]: (actionsByTranslationUrlByUserUrl[userUrl]?.[translationUrl]?.actions[action] ?? 0) + 1, + }, + }, + }; + }); + + return { userUrlsByTranslationUrl, actionsByTranslationUrlByUserUrl, actions }; +}; + +const getUsernameByUserUrlMap = async ( + userUrlsByTranslationUrl: UserUrlsByTranslationUrl, + authToken: string, +): Promise> => + Object.fromEntries( + (await Promise.all( + Object.values(userUrlsByTranslationUrl) + .flat() + .map(async (userUrl) => [userUrl, await getUsername(userUrl, authToken)]), + )) satisfies [string, string][], + ); + +const getLanguageNameByTranslationUrl = async ( + userUrlsByTranslationUrl: UserUrlsByTranslationUrl, + authToken: string, +): Promise => + Object.fromEntries( + (await Promise.all( + Object.keys(userUrlsByTranslationUrl) + .flat() + .map(async (translationUrl) => [translationUrl, await getLanguageName(translationUrl, authToken)]), + )) satisfies [string, string][], + ); + +const getUserContributionByLanguage = ( + userUrlsByTranslationUrl: UserUrlsByTranslationUrl, + languageNameByTranslationUrl: LanguageNameByTranslationUrl, + usernameByUserUrl: UsernameByUserUrl, + actionsByTranslationUrlByUserUrl: ActionByTranslationUrlByUserUrl, +) => + Object.fromEntries( + Object.entries(userUrlsByTranslationUrl).map(([translationUrl, userUrls]) => { + const languageName = languageNameByTranslationUrl[translationUrl]; + const contributors = userUrls + .map((userUrl) => { + const username = usernameByUserUrl[userUrl]; + const contributionCount = actionsByTranslationUrlByUserUrl[userUrl][translationUrl].count; + + return { username, contributionCount }; + }) + .sort((userA, userB) => userB.contributionCount - userA.contributionCount); + + return [languageName, contributors]; + }), + ); + +const getContributorsForRange = async (url: string, authToken: string): Promise => { + const weblateChanges = await fetchWeblateChanges(url, authToken); + + const { userUrlsByTranslationUrl, actionsByTranslationUrlByUserUrl } = + extractContributionInfoFromChanges(weblateChanges); + + const usernameByUserUrl = await getUsernameByUserUrlMap(userUrlsByTranslationUrl, authToken); + const languageNameByTranslationUrl = await getLanguageNameByTranslationUrl(userUrlsByTranslationUrl, authToken); + + return getUserContributionByLanguage( + userUrlsByTranslationUrl, + languageNameByTranslationUrl, + usernameByUserUrl, + actionsByTranslationUrlByUserUrl, + ); +}; + +const getLanguageLinesFromChangelog = (): Promise => { + const fileReadPromise = new ControlledPromise(); + + const changelogFile = path.resolve(__dirname, '../../CHANGELOG.md'); + const fileStream = fs.createReadStream(changelogFile); // Replace 'input.txt' with your file path + const fileLineReader = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity, // Recognize all line breaks (CR LF) as a single line break + }); + + const translationRelevantLines: string[] = []; + let isTranslationSection = false; + fileLineReader.on('line', (line) => { + if (line === '## Translations') { + isTranslationSection = true; + return; + } + + if (isTranslationSection && line === '## Full Changelog') { + isTranslationSection = false; + return; + } + + if (!isTranslationSection) { + return; + } + + if (!line.startsWith('-')) { + return; + } + + translationRelevantLines.push(line); + }); + + fileLineReader.on('close', () => { + fileReadPromise.resolve(translationRelevantLines); + }); + + fileLineReader.on('error', (error) => { + fileReadPromise.reject(error); + }); + + return fileReadPromise.promise; +}; + +const getKnownContributorsByLanguage = async (): Promise> => { + const changelogLanguageLines = await getLanguageLinesFromChangelog(); + + const contributorByLanguage: Record = {}; + changelogLanguageLines.forEach((languageLine) => { + const languageChangeRegex = /^- (.*) \(by (.*)\)$/g; + const languageChangeRegexMatch = [...languageLine.matchAll(languageChangeRegex)][0]; + + const language = languageChangeRegexMatch[1]; + const contributors = languageChangeRegexMatch[2].split(', '); + + const knownContributors = contributorByLanguage[language] ?? []; + contributorByLanguage[language] = [...new Set([...knownContributors, ...contributors])]; + }); + + return contributorByLanguage; +}; + +const createTranslationChangelog = async (): Promise => { + const url = `https://hosted.weblate.org/api/components/suwayomi/suwayomi-webui/changes/?timestamp_after=${timestampAfter}×tamp_before=${timestampBefore}&${creditRelevantActions.map((action) => `action=${action}`).join('&')}`; + + const contributorsForRange = await getContributorsForRange(url, tokens.weblateToken); + const knownContributorsByLanguage = await getKnownContributorsByLanguage(); + + const contributionInfo: { language: string; contributors: string[] }[] = Object.entries(contributorsForRange).map( + ([language, languageContributors]) => { + const validContributors = languageContributors.filter( + ({ username, contributionCount }) => + knownContributorsByLanguage[language]?.includes(username) || contributionCount >= 10, + ); + const contributors = validContributors.length ? validContributors : [languageContributors[0]]; + + return { + language, + contributors: contributors.map(({ username }) => username), + }; + }, + ); + + const languageByNewState = Object.groupBy( + contributionInfo, + ({ language }) => `${knownContributorsByLanguage[language] == null}`, + ); + + const newLanguages = languageByNewState.true ?? []; + const updatedLanguages = languageByNewState.false ?? []; + + console.log( + '## Translations\n' + + 'Feel free to translate the project on [Weblate](https://hosted.weblate.org/projects/suwayomi/suwayomi-webui/)\n' + + '\n' + + 'Thank you for your contribution to the translation of the project.\n', + ); + + console.log('### Added'); + newLanguages.forEach(({ language, contributors }) => { + console.log(`- ${language} (by ${contributors.join(', ')})`); + }); + console.log(''); + console.log('### Updated'); + updatedLanguages.forEach(({ language, contributors }) => { + console.log(`- ${language} (by ${contributors.join(', ')})`); + }); +}; + +createTranslationChangelog(); diff --git a/tools/scripts/github_token.template.json b/tools/scripts/github_token.template.json deleted file mode 100644 index 5ae81604..00000000 --- a/tools/scripts/github_token.template.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "token": "" -} \ No newline at end of file diff --git a/tools/scripts/token.template.json b/tools/scripts/token.template.json new file mode 100644 index 00000000..1508021a --- /dev/null +++ b/tools/scripts/token.template.json @@ -0,0 +1,4 @@ +{ + "githubToken": "", + "weblateToken": "" +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index f16adff6..b8b7c8ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5256,7 +5256,7 @@ json5@^1.0.2: dependencies: minimist "^1.2.0" -json5@^2.2.3: +json5@^2.2.2, json5@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== @@ -6850,6 +6850,15 @@ tsconfck@^3.0.3: resolved "https://registry.yarnpkg.com/tsconfck/-/tsconfck-3.0.3.tgz#d9bda0e87d05b1c360e996c9050473c7e6f8084f" integrity sha512-4t0noZX9t6GcPTfBAbIbbIU4pfpCwh0ueq3S4O/5qXI1VwK1outmxhe9dOiEWqMz3MW2LKgDTpqWV+37IWuVbA== +tsconfig-paths@4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz#ef78e19039133446d244beac0fd6a1632e2d107c" + integrity sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg== + dependencies: + json5 "^2.2.2" + minimist "^1.2.6" + strip-bom "^3.0.0" + tsconfig-paths@^3.15.0: version "3.15.0" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz#5299ec605e55b1abb23ec939ef15edaf483070d4"