refactor(i18n): Translation's lint and load (#31343)
Tasso Evangelista
2 years ago
committed by
GitHub
parent
c5693fb8c8
commit
eecf782737
@ -1,109 +0,0 @@ |
||||
const fs = require('fs'); |
||||
const path = require('path'); |
||||
|
||||
const fg = require('fast-glob'); |
||||
|
||||
const regexVar = /__[a-zA-Z_]+__/g; |
||||
|
||||
const validateKeys = (json, usedKeys) => |
||||
usedKeys |
||||
.filter(({ key }) => typeof json[key] !== 'undefined') |
||||
.reduce((prev, cur) => { |
||||
const { key, replaces } = cur; |
||||
|
||||
const miss = replaces.filter((replace) => json[key] && json[key].indexOf(replace) === -1); |
||||
|
||||
if (miss.length > 0) { |
||||
prev.push({ key, miss }); |
||||
} |
||||
|
||||
return prev; |
||||
}, []); |
||||
|
||||
const removeMissingKeys = (i18nFiles, usedKeys) => { |
||||
i18nFiles.forEach((file) => { |
||||
const json = JSON.parse(fs.readFileSync(file, 'utf8')); |
||||
if (Object.keys(json).length === 0) { |
||||
return; |
||||
} |
||||
|
||||
validateKeys(json, usedKeys).forEach(({ key }) => { |
||||
json[key] = null; |
||||
}); |
||||
|
||||
fs.writeFileSync(file, JSON.stringify(json, null, 2)); |
||||
}); |
||||
}; |
||||
|
||||
const checkUniqueKeys = (content, json, filename) => { |
||||
const matchKeys = content.matchAll(/^\s+"([^"]+)"/gm); |
||||
|
||||
const allKeys = [...matchKeys]; |
||||
|
||||
if (allKeys.length !== Object.keys(json).length) { |
||||
throw new Error(`Duplicated keys found on file ${filename}`); |
||||
} |
||||
}; |
||||
|
||||
const validate = (i18nFiles, usedKeys) => { |
||||
const totalErrors = i18nFiles.reduce((errors, file) => { |
||||
const content = fs.readFileSync(file, 'utf8'); |
||||
const json = JSON.parse(content); |
||||
|
||||
checkUniqueKeys(content, json, file); |
||||
|
||||
// console.log('json, usedKeys2', json, usedKeys);
|
||||
|
||||
const result = validateKeys(json, usedKeys); |
||||
|
||||
if (result.length === 0) { |
||||
return errors; |
||||
} |
||||
|
||||
console.log('\n## File', file, `(${result.length} errors)`); |
||||
|
||||
result.forEach(({ key, miss }) => { |
||||
console.log('\n- Key:', key, '\n Missing variables:', miss.join(', ')); |
||||
}); |
||||
|
||||
return errors + result.length; |
||||
}, 0); |
||||
|
||||
if (totalErrors > 0) { |
||||
throw new Error(`\n${totalErrors} errors found`); |
||||
} |
||||
}; |
||||
|
||||
const checkFiles = async (sourcePath, sourceFile, fix = false) => { |
||||
const content = fs.readFileSync(path.join(sourcePath, sourceFile), 'utf8'); |
||||
const sourceContent = JSON.parse(content); |
||||
|
||||
checkUniqueKeys(content, sourceContent, sourceFile); |
||||
|
||||
const usedKeys = Object.entries(sourceContent).map(([key, value]) => { |
||||
const replaces = value.match(regexVar); |
||||
return { |
||||
key, |
||||
replaces, |
||||
}; |
||||
}); |
||||
|
||||
const keysWithInterpolation = usedKeys.filter(({ replaces }) => !!replaces); |
||||
|
||||
const i18nFiles = await fg([`${sourcePath}/**/*.i18n.json`]); |
||||
|
||||
if (fix) { |
||||
return removeMissingKeys(i18nFiles, keysWithInterpolation); |
||||
} |
||||
|
||||
validate(i18nFiles, keysWithInterpolation); |
||||
}; |
||||
|
||||
(async () => { |
||||
try { |
||||
await checkFiles('./packages/rocketchat-i18n/i18n', 'en.i18n.json', process.argv[2] === '--fix'); |
||||
} catch (e) { |
||||
console.error(e); |
||||
process.exit(1); |
||||
} |
||||
})(); |
||||
@ -0,0 +1,249 @@ |
||||
import type { PathLike } from 'node:fs'; |
||||
import { readFile, writeFile } from 'node:fs/promises'; |
||||
import { join } from 'node:path'; |
||||
import { inspect } from 'node:util'; |
||||
|
||||
import fg from 'fast-glob'; |
||||
import i18next from 'i18next'; |
||||
import supportsColor from 'supports-color'; |
||||
|
||||
const hasDuplicatedKeys = (content: string, json: Record<string, string>) => { |
||||
const matchKeys = content.matchAll(/^\s+"([^"]+)"/gm); |
||||
|
||||
const allKeys = [...matchKeys]; |
||||
|
||||
return allKeys.length !== Object.keys(json).length; |
||||
}; |
||||
|
||||
const parseFile = async (path: PathLike) => { |
||||
const content = await readFile(path, 'utf-8'); |
||||
let json: Record<string, string>; |
||||
try { |
||||
json = JSON.parse(content); |
||||
} catch (e) { |
||||
if (e instanceof SyntaxError) { |
||||
const matches = /^Unexpected token .* in JSON at position (\d+)$/.exec(e.message); |
||||
|
||||
if (matches) { |
||||
const [, positionStr] = matches; |
||||
const position = parseInt(positionStr, 10); |
||||
const line = content.slice(0, position).split('\n').length; |
||||
const column = position - content.slice(0, position).lastIndexOf('\n'); |
||||
throw new SyntaxError(`Invalid JSON on file ${path}:${line}:${column}`); |
||||
} |
||||
} |
||||
throw new SyntaxError(`Invalid JSON on file ${path}: ${e.message}`); |
||||
} |
||||
|
||||
if (hasDuplicatedKeys(content, json)) { |
||||
throw new SyntaxError(`Duplicated keys found on file ${path}`); |
||||
} |
||||
|
||||
return json; |
||||
}; |
||||
|
||||
const insertTranslation = (json: Record<string, string>, refKey: string, [key, value]: [key: string, value: string]) => { |
||||
const entries = Object.entries(json); |
||||
|
||||
const refIndex = entries.findIndex(([entryKey]) => entryKey === refKey); |
||||
|
||||
if (refIndex === -1) { |
||||
throw new Error(`Reference key ${refKey} not found`); |
||||
} |
||||
|
||||
const movingEntries = entries.slice(refIndex + 1); |
||||
|
||||
for (const [key] of movingEntries) { |
||||
delete json[key]; |
||||
} |
||||
|
||||
json[key] = value; |
||||
|
||||
for (const [key, value] of movingEntries) { |
||||
json[key] = value; |
||||
} |
||||
}; |
||||
|
||||
const persistFile = async (path: PathLike, json: Record<string, string>) => { |
||||
const content = JSON.stringify(json, null, 2); |
||||
|
||||
await writeFile(path, content, 'utf-8'); |
||||
}; |
||||
|
||||
const oldPlaceholderFormat = /__([a-zA-Z_]+)__/g; |
||||
|
||||
const checkPlaceholdersFormat = async ({ json, path, fix = false }: { json: Record<string, string>; path: PathLike; fix?: boolean }) => { |
||||
const outdatedKeys = Object.entries(json) |
||||
.map(([key, value]) => ({ |
||||
key, |
||||
value, |
||||
placeholders: value.match(oldPlaceholderFormat), |
||||
})) |
||||
.filter((outdatedKey): outdatedKey is { key: string; value: string; placeholders: RegExpMatchArray } => !!outdatedKey.placeholders); |
||||
|
||||
if (outdatedKeys.length > 0) { |
||||
const message = `Outdated placeholder format on file ${path}: ${inspect(outdatedKeys, { colors: !!supportsColor.stdout })}`; |
||||
|
||||
if (fix) { |
||||
console.warn(message); |
||||
|
||||
for (const { key, value } of outdatedKeys) { |
||||
const newValue = value.replace(oldPlaceholderFormat, (_, name) => `{{${name}}}`); |
||||
|
||||
json[key] = newValue; |
||||
} |
||||
|
||||
await persistFile(path, json); |
||||
|
||||
return; |
||||
} |
||||
|
||||
throw new Error(message); |
||||
} |
||||
}; |
||||
|
||||
export const extractSingularKeys = (json: Record<string, string>, lng: string) => { |
||||
if (!i18next.isInitialized) { |
||||
i18next.init({ initImmediate: false }); |
||||
} |
||||
|
||||
const pluralSuffixes = i18next.services.pluralResolver.getSuffixes(lng) as string[]; |
||||
|
||||
const singularKeys = new Set( |
||||
Object.keys(json).map((key) => { |
||||
for (const pluralSuffix of pluralSuffixes) { |
||||
if (key.endsWith(pluralSuffix)) { |
||||
return key.slice(0, -pluralSuffix.length); |
||||
} |
||||
} |
||||
|
||||
return key; |
||||
}), |
||||
); |
||||
|
||||
return [singularKeys, pluralSuffixes] as const; |
||||
}; |
||||
|
||||
const checkMissingPlurals = async ({ |
||||
json, |
||||
path, |
||||
lng, |
||||
fix = false, |
||||
}: { |
||||
json: Record<string, string>; |
||||
path: PathLike; |
||||
lng: string; |
||||
fix?: boolean; |
||||
}) => { |
||||
const [singularKeys, pluralSuffixes] = extractSingularKeys(json, lng); |
||||
|
||||
const missingPluralKeys: { singularKey: string; existing: string[]; missing: string[] }[] = []; |
||||
|
||||
for (const singularKey of singularKeys) { |
||||
if (singularKey in json) { |
||||
continue; |
||||
} |
||||
|
||||
const pluralKeys = pluralSuffixes.map((suffix) => `${singularKey}${suffix}`); |
||||
|
||||
const existing = pluralKeys.filter((key) => key in json); |
||||
const missing = pluralKeys.filter((key) => !(key in json)); |
||||
|
||||
if (missing.length > 0) { |
||||
missingPluralKeys.push({ singularKey, existing, missing }); |
||||
} |
||||
} |
||||
|
||||
if (missingPluralKeys.length > 0) { |
||||
const message = `Missing plural keys on file ${path}: ${inspect(missingPluralKeys, { colors: !!supportsColor.stdout })}`; |
||||
|
||||
if (fix) { |
||||
console.warn(message); |
||||
|
||||
for (const { existing, missing } of missingPluralKeys) { |
||||
for (const missingKey of missing) { |
||||
const refKey = existing.slice(-1)[0]; |
||||
const value = json[refKey]; |
||||
insertTranslation(json, refKey, [missingKey, value]); |
||||
} |
||||
} |
||||
|
||||
await persistFile(path, json); |
||||
|
||||
return; |
||||
} |
||||
|
||||
throw new Error(message); |
||||
} |
||||
}; |
||||
|
||||
const checkExceedingKeys = async ({ |
||||
json, |
||||
path, |
||||
lng, |
||||
sourceJson, |
||||
sourceLng, |
||||
fix = false, |
||||
}: { |
||||
json: Record<string, string>; |
||||
path: PathLike; |
||||
lng: string; |
||||
sourceJson: Record<string, string>; |
||||
sourceLng: string; |
||||
fix?: boolean; |
||||
}) => { |
||||
const [singularKeys] = extractSingularKeys(json, lng); |
||||
const [sourceSingularKeys] = extractSingularKeys(sourceJson, sourceLng); |
||||
|
||||
const exceedingKeys = [...singularKeys].filter((key) => !sourceSingularKeys.has(key)); |
||||
|
||||
if (exceedingKeys.length > 0) { |
||||
const message = `Exceeding keys on file ${path}: ${inspect(exceedingKeys, { colors: !!supportsColor.stdout })}`; |
||||
|
||||
if (fix) { |
||||
for (const key of exceedingKeys) { |
||||
delete json[key]; |
||||
} |
||||
|
||||
await persistFile(path, json); |
||||
|
||||
return; |
||||
} |
||||
|
||||
throw new Error(message); |
||||
} |
||||
}; |
||||
|
||||
const checkFiles = async (sourceDirPath: string, sourceLng: string, fix = false) => { |
||||
const sourcePath = join(sourceDirPath, `${sourceLng}.i18n.json`); |
||||
const sourceJson = await parseFile(sourcePath); |
||||
|
||||
await checkPlaceholdersFormat({ json: sourceJson, path: sourcePath, fix }); |
||||
await checkMissingPlurals({ json: sourceJson, path: sourcePath, lng: sourceLng, fix }); |
||||
|
||||
const i18nFiles = await fg([join(sourceDirPath, `**/*.i18n.json`), `!${sourcePath}`]); |
||||
|
||||
const languageFileRegex = /\/([^\/]*?).i18n.json$/; |
||||
const translations = await Promise.all( |
||||
i18nFiles.map(async (path) => { |
||||
const lng = languageFileRegex.exec(path)?.[1]; |
||||
if (!lng) { |
||||
throw new Error(`Invalid language file path ${path}`); |
||||
} |
||||
|
||||
return { path, json: await parseFile(path), lng }; |
||||
}), |
||||
); |
||||
|
||||
for await (const { path, json, lng } of translations) { |
||||
await checkPlaceholdersFormat({ json, path, fix }); |
||||
await checkMissingPlurals({ json, path, lng, fix }); |
||||
await checkExceedingKeys({ json, path, lng, sourceJson, sourceLng, fix }); |
||||
} |
||||
}; |
||||
|
||||
const fix = process.argv[2] === '--fix'; |
||||
checkFiles('./packages/rocketchat-i18n/i18n', 'en', fix).catch((e) => { |
||||
console.error(e); |
||||
process.exit(1); |
||||
}); |
||||
@ -1,28 +0,0 @@ |
||||
import { settings } from '../../../app/settings/client'; |
||||
import { i18n } from '../../../app/utils/lib/i18n'; |
||||
|
||||
const parseToJSON = (customTranslations: string) => { |
||||
try { |
||||
return JSON.parse(customTranslations); |
||||
} catch (e) { |
||||
return false; |
||||
} |
||||
}; |
||||
|
||||
export const applyCustomTranslations = (): void => { |
||||
const customTranslations: string | undefined = settings.get('Custom_Translations'); |
||||
|
||||
if (!customTranslations || !parseToJSON(customTranslations)) { |
||||
return; |
||||
} |
||||
|
||||
try { |
||||
const parsedCustomTranslations: Record<string, unknown> = JSON.parse(customTranslations); |
||||
|
||||
for (const [lang, translations] of Object.entries(parsedCustomTranslations)) { |
||||
i18n.addResourceBundle(lang, 'core', translations); |
||||
} |
||||
} catch (e) { |
||||
console.error('Invalid setting Custom_Translations', e); |
||||
} |
||||
}; |
||||
@ -1 +0,0 @@ |
||||
../../../apps/meteor/private/i18n/en.i18n.json |
||||
Loading…
Reference in new issue