dify
This commit is contained in:
496
dify/web/i18n-config/check-i18n.js
Normal file
496
dify/web/i18n-config/check-i18n.js
Normal file
@@ -0,0 +1,496 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const vm = require('node:vm')
|
||||
const transpile = require('typescript').transpile
|
||||
|
||||
const targetLanguage = 'en-US'
|
||||
const data = require('./languages.json')
|
||||
const languages = data.languages.filter(language => language.supported).map(language => language.value)
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
files: [],
|
||||
languages: [],
|
||||
autoRemove: false,
|
||||
help: false,
|
||||
errors: [],
|
||||
}
|
||||
|
||||
const collectValues = (startIndex) => {
|
||||
const values = []
|
||||
let cursor = startIndex + 1
|
||||
while (cursor < argv.length && !argv[cursor].startsWith('--')) {
|
||||
const value = argv[cursor].trim()
|
||||
if (value) values.push(value)
|
||||
cursor++
|
||||
}
|
||||
return { values, nextIndex: cursor - 1 }
|
||||
}
|
||||
|
||||
const validateList = (values, flag) => {
|
||||
if (!values.length) {
|
||||
args.errors.push(`${flag} requires at least one value. Example: ${flag} app billing`)
|
||||
return false
|
||||
}
|
||||
|
||||
const invalid = values.find(value => value.includes(','))
|
||||
if (invalid) {
|
||||
args.errors.push(`${flag} expects space-separated values. Example: ${flag} app billing`)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
for (let index = 2; index < argv.length; index++) {
|
||||
const arg = argv[index]
|
||||
|
||||
if (arg === '--auto-remove') {
|
||||
args.autoRemove = true
|
||||
continue
|
||||
}
|
||||
|
||||
if (arg === '--help' || arg === '-h') {
|
||||
args.help = true
|
||||
break
|
||||
}
|
||||
|
||||
if (arg.startsWith('--file=')) {
|
||||
args.errors.push('--file expects space-separated values. Example: --file app billing')
|
||||
continue
|
||||
}
|
||||
|
||||
if (arg === '--file') {
|
||||
const { values, nextIndex } = collectValues(index)
|
||||
if (validateList(values, '--file'))
|
||||
args.files.push(...values)
|
||||
index = nextIndex
|
||||
continue
|
||||
}
|
||||
|
||||
if (arg.startsWith('--lang=')) {
|
||||
args.errors.push('--lang expects space-separated values. Example: --lang zh-Hans ja-JP')
|
||||
continue
|
||||
}
|
||||
|
||||
if (arg === '--lang') {
|
||||
const { values, nextIndex } = collectValues(index)
|
||||
if (validateList(values, '--lang'))
|
||||
args.languages.push(...values)
|
||||
index = nextIndex
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Usage: pnpm run check-i18n [options]
|
||||
|
||||
Options:
|
||||
--file <name...> Check only specific files; provide space-separated names and repeat --file if needed
|
||||
--lang <locale> Check only specific locales; provide space-separated locales and repeat --lang if needed
|
||||
--auto-remove Remove extra keys automatically
|
||||
-h, --help Show help
|
||||
|
||||
Examples:
|
||||
pnpm run check-i18n -- --file app billing --lang zh-Hans ja-JP
|
||||
pnpm run check-i18n -- --auto-remove
|
||||
`)
|
||||
}
|
||||
|
||||
async function getKeysFromLanguage(language) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const folderPath = path.resolve(__dirname, '../i18n', language)
|
||||
const allKeys = []
|
||||
fs.readdir(folderPath, (err, files) => {
|
||||
if (err) {
|
||||
console.error('Error reading folder:', err)
|
||||
reject(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Filter only .ts and .js files
|
||||
const translationFiles = files.filter(file => /\.(ts|js)$/.test(file))
|
||||
|
||||
translationFiles.forEach((file) => {
|
||||
const filePath = path.join(folderPath, file)
|
||||
const fileName = file.replace(/\.[^/.]+$/, '') // Remove file extension
|
||||
const camelCaseFileName = fileName.replace(/[-_](.)/g, (_, c) =>
|
||||
c.toUpperCase(),
|
||||
) // Convert to camel case
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf8')
|
||||
|
||||
// Create a safer module environment for vm
|
||||
const moduleExports = {}
|
||||
const context = {
|
||||
exports: moduleExports,
|
||||
module: { exports: moduleExports },
|
||||
require,
|
||||
console,
|
||||
__filename: filePath,
|
||||
__dirname: folderPath,
|
||||
}
|
||||
|
||||
// Use vm.runInNewContext instead of eval for better security
|
||||
vm.runInNewContext(transpile(content), context)
|
||||
|
||||
// Extract the translation object
|
||||
const translationObj = moduleExports.default || moduleExports
|
||||
|
||||
if(!translationObj || typeof translationObj !== 'object') {
|
||||
console.error(`Error parsing file: ${filePath}`)
|
||||
reject(new Error(`Error parsing file: ${filePath}`))
|
||||
return
|
||||
}
|
||||
|
||||
const nestedKeys = []
|
||||
const iterateKeys = (obj, prefix = '') => {
|
||||
for (const key in obj) {
|
||||
const nestedKey = prefix ? `${prefix}.${key}` : key
|
||||
if (typeof obj[key] === 'object' && obj[key] !== null && !Array.isArray(obj[key])) {
|
||||
// This is an object (but not array), recurse into it but don't add it as a key
|
||||
iterateKeys(obj[key], nestedKey)
|
||||
}
|
||||
else {
|
||||
// This is a leaf node (string, number, boolean, array, etc.), add it as a key
|
||||
nestedKeys.push(nestedKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
iterateKeys(translationObj)
|
||||
|
||||
// Fixed: accumulate keys instead of overwriting
|
||||
const fileKeys = nestedKeys.map(key => `${camelCaseFileName}.${key}`)
|
||||
allKeys.push(...fileKeys)
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`Error processing file ${filePath}:`, error.message)
|
||||
reject(error)
|
||||
}
|
||||
})
|
||||
resolve(allKeys)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function removeKeysFromObject(obj, keysToRemove, prefix = '') {
|
||||
let modified = false
|
||||
for (const key in obj) {
|
||||
const fullKey = prefix ? `${prefix}.${key}` : key
|
||||
|
||||
if (keysToRemove.includes(fullKey)) {
|
||||
delete obj[key]
|
||||
modified = true
|
||||
console.log(`🗑️ Removed key: ${fullKey}`)
|
||||
}
|
||||
else if (typeof obj[key] === 'object' && obj[key] !== null) {
|
||||
const subModified = removeKeysFromObject(obj[key], keysToRemove, fullKey)
|
||||
modified = modified || subModified
|
||||
}
|
||||
}
|
||||
return modified
|
||||
}
|
||||
|
||||
async function removeExtraKeysFromFile(language, fileName, extraKeys) {
|
||||
const filePath = path.resolve(__dirname, '../i18n', language, `${fileName}.ts`)
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.log(`⚠️ File not found: ${filePath}`)
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
// Filter keys that belong to this file
|
||||
const camelCaseFileName = fileName.replace(/[-_](.)/g, (_, c) => c.toUpperCase())
|
||||
const fileSpecificKeys = extraKeys
|
||||
.filter(key => key.startsWith(`${camelCaseFileName}.`))
|
||||
.map(key => key.substring(camelCaseFileName.length + 1)) // Remove file prefix
|
||||
|
||||
if (fileSpecificKeys.length === 0)
|
||||
return false
|
||||
|
||||
console.log(`🔄 Processing file: ${filePath}`)
|
||||
|
||||
// Read the original file content
|
||||
const content = fs.readFileSync(filePath, 'utf8')
|
||||
const lines = content.split('\n')
|
||||
|
||||
let modified = false
|
||||
const linesToRemove = []
|
||||
|
||||
// Find lines to remove for each key (including multiline values)
|
||||
for (const keyToRemove of fileSpecificKeys) {
|
||||
const keyParts = keyToRemove.split('.')
|
||||
let targetLineIndex = -1
|
||||
const linesToRemoveForKey = []
|
||||
|
||||
// Build regex pattern for the exact key path
|
||||
if (keyParts.length === 1) {
|
||||
// Simple key at root level like "pickDate: 'value'"
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i]
|
||||
const simpleKeyPattern = new RegExp(`^\\s*${keyParts[0]}\\s*:`)
|
||||
if (simpleKeyPattern.test(line)) {
|
||||
targetLineIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Nested key - need to find the exact path
|
||||
const currentPath = []
|
||||
let braceDepth = 0
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i]
|
||||
const trimmedLine = line.trim()
|
||||
|
||||
// Track current object path
|
||||
const keyMatch = trimmedLine.match(/^(\w+)\s*:\s*{/)
|
||||
if (keyMatch) {
|
||||
currentPath.push(keyMatch[1])
|
||||
braceDepth++
|
||||
}
|
||||
else if (trimmedLine === '},' || trimmedLine === '}') {
|
||||
if (braceDepth > 0) {
|
||||
braceDepth--
|
||||
currentPath.pop()
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this line matches our target key
|
||||
const leafKeyMatch = trimmedLine.match(/^(\w+)\s*:/)
|
||||
if (leafKeyMatch) {
|
||||
const fullPath = [...currentPath, leafKeyMatch[1]]
|
||||
const fullPathString = fullPath.join('.')
|
||||
|
||||
if (fullPathString === keyToRemove) {
|
||||
targetLineIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (targetLineIndex !== -1) {
|
||||
linesToRemoveForKey.push(targetLineIndex)
|
||||
|
||||
// Check if this is a multiline key-value pair
|
||||
const keyLine = lines[targetLineIndex]
|
||||
const trimmedKeyLine = keyLine.trim()
|
||||
|
||||
// If key line ends with ":" (not ":", "{ " or complete value), it's likely multiline
|
||||
if (trimmedKeyLine.endsWith(':') && !trimmedKeyLine.includes('{') && !trimmedKeyLine.match(/:\s*['"`]/)) {
|
||||
// Find the value lines that belong to this key
|
||||
let currentLine = targetLineIndex + 1
|
||||
let foundValue = false
|
||||
|
||||
while (currentLine < lines.length) {
|
||||
const line = lines[currentLine]
|
||||
const trimmed = line.trim()
|
||||
|
||||
// Skip empty lines
|
||||
if (trimmed === '') {
|
||||
currentLine++
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if this line starts a new key (indicates end of current value)
|
||||
if (trimmed.match(/^\w+\s*:/))
|
||||
break
|
||||
|
||||
// Check if this line is part of the value
|
||||
if (trimmed.startsWith('\'') || trimmed.startsWith('"') || trimmed.startsWith('`') || foundValue) {
|
||||
linesToRemoveForKey.push(currentLine)
|
||||
foundValue = true
|
||||
|
||||
// Check if this line ends the value (ends with quote and comma/no comma)
|
||||
if ((trimmed.endsWith('\',') || trimmed.endsWith('",') || trimmed.endsWith('`,')
|
||||
|| trimmed.endsWith('\'') || trimmed.endsWith('"') || trimmed.endsWith('`'))
|
||||
&& !trimmed.startsWith('//'))
|
||||
break
|
||||
}
|
||||
else {
|
||||
break
|
||||
}
|
||||
|
||||
currentLine++
|
||||
}
|
||||
}
|
||||
|
||||
linesToRemove.push(...linesToRemoveForKey)
|
||||
console.log(`🗑️ Found key to remove: ${keyToRemove} at line ${targetLineIndex + 1}${linesToRemoveForKey.length > 1 ? ` (multiline, ${linesToRemoveForKey.length} lines)` : ''}`)
|
||||
modified = true
|
||||
}
|
||||
else {
|
||||
console.log(`⚠️ Could not find key: ${keyToRemove}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
// Remove duplicates and sort in reverse order to maintain correct indices
|
||||
const uniqueLinesToRemove = [...new Set(linesToRemove)].sort((a, b) => b - a)
|
||||
|
||||
for (const lineIndex of uniqueLinesToRemove) {
|
||||
const line = lines[lineIndex]
|
||||
console.log(`🗑️ Removing line ${lineIndex + 1}: ${line.trim()}`)
|
||||
lines.splice(lineIndex, 1)
|
||||
|
||||
// Also remove trailing comma from previous line if it exists and the next line is a closing brace
|
||||
if (lineIndex > 0 && lineIndex < lines.length) {
|
||||
const prevLine = lines[lineIndex - 1]
|
||||
const nextLine = lines[lineIndex] ? lines[lineIndex].trim() : ''
|
||||
|
||||
if (prevLine.trim().endsWith(',') && (nextLine.startsWith('}') || nextLine === ''))
|
||||
lines[lineIndex - 1] = prevLine.replace(/,\s*$/, '')
|
||||
}
|
||||
}
|
||||
|
||||
// Write back to file
|
||||
const newContent = lines.join('\n')
|
||||
fs.writeFileSync(filePath, newContent)
|
||||
console.log(`💾 Updated file: ${filePath}`)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`Error processing file ${filePath}:`, error.message)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Add command line argument support
|
||||
const args = parseArgs(process.argv)
|
||||
const targetFiles = Array.from(new Set(args.files))
|
||||
const targetLangs = Array.from(new Set(args.languages))
|
||||
const autoRemove = args.autoRemove
|
||||
|
||||
async function main() {
|
||||
const compareKeysCount = async () => {
|
||||
let hasDiff = false
|
||||
const allTargetKeys = await getKeysFromLanguage(targetLanguage)
|
||||
|
||||
// Filter target keys by file if specified
|
||||
const camelTargetFiles = targetFiles.map(file => file.replace(/[-_](.)/g, (_, c) => c.toUpperCase()))
|
||||
const targetKeys = targetFiles.length
|
||||
? allTargetKeys.filter(key => camelTargetFiles.some(file => key.startsWith(`${file}.`)))
|
||||
: allTargetKeys
|
||||
|
||||
// Filter languages by target language if specified
|
||||
const languagesToProcess = targetLangs.length ? targetLangs : languages
|
||||
|
||||
const allLanguagesKeys = await Promise.all(languagesToProcess.map(language => getKeysFromLanguage(language)))
|
||||
|
||||
// Filter language keys by file if specified
|
||||
const languagesKeys = targetFiles.length
|
||||
? allLanguagesKeys.map(keys => keys.filter(key => camelTargetFiles.some(file => key.startsWith(`${file}.`))))
|
||||
: allLanguagesKeys
|
||||
|
||||
const keysCount = languagesKeys.map(keys => keys.length)
|
||||
const targetKeysCount = targetKeys.length
|
||||
|
||||
const comparison = languagesToProcess.reduce((result, language, index) => {
|
||||
const languageKeysCount = keysCount[index]
|
||||
const difference = targetKeysCount - languageKeysCount
|
||||
result[language] = difference
|
||||
return result
|
||||
}, {})
|
||||
|
||||
console.log(comparison)
|
||||
|
||||
// Print missing keys and extra keys
|
||||
for (let index = 0; index < languagesToProcess.length; index++) {
|
||||
const language = languagesToProcess[index]
|
||||
const languageKeys = languagesKeys[index]
|
||||
const missingKeys = targetKeys.filter(key => !languageKeys.includes(key))
|
||||
const extraKeys = languageKeys.filter(key => !targetKeys.includes(key))
|
||||
|
||||
console.log(`Missing keys in ${language}:`, missingKeys)
|
||||
if (missingKeys.length > 0)
|
||||
hasDiff = true
|
||||
|
||||
// Show extra keys only when there are extra keys (negative difference)
|
||||
if (extraKeys.length > 0) {
|
||||
console.log(`Extra keys in ${language} (not in ${targetLanguage}):`, extraKeys)
|
||||
|
||||
// Auto-remove extra keys if flag is set
|
||||
if (autoRemove) {
|
||||
console.log(`\n🤖 Auto-removing extra keys from ${language}...`)
|
||||
|
||||
// Get all translation files
|
||||
const i18nFolder = path.resolve(__dirname, '../i18n', language)
|
||||
const files = fs.readdirSync(i18nFolder)
|
||||
.filter(file => /\.ts$/.test(file))
|
||||
.map(file => file.replace(/\.ts$/, ''))
|
||||
.filter(f => targetFiles.length === 0 || targetFiles.includes(f))
|
||||
|
||||
let totalRemoved = 0
|
||||
for (const fileName of files) {
|
||||
const removed = await removeExtraKeysFromFile(language, fileName, extraKeys)
|
||||
if (removed) totalRemoved++
|
||||
}
|
||||
|
||||
console.log(`✅ Auto-removal completed for ${language}. Modified ${totalRemoved} files.`)
|
||||
}
|
||||
else {
|
||||
hasDiff = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return hasDiff
|
||||
}
|
||||
|
||||
console.log('🚀 Starting check-i18n script...')
|
||||
if (targetFiles.length)
|
||||
console.log(`📁 Checking files: ${targetFiles.join(', ')}`)
|
||||
|
||||
if (targetLangs.length)
|
||||
console.log(`🌍 Checking languages: ${targetLangs.join(', ')}`)
|
||||
|
||||
if (autoRemove)
|
||||
console.log('🤖 Auto-remove mode: ENABLED')
|
||||
|
||||
const hasDiff = await compareKeysCount()
|
||||
if (hasDiff) {
|
||||
console.error('\n❌ i18n keys are not aligned. Fix issues above.')
|
||||
process.exitCode = 1
|
||||
}
|
||||
else {
|
||||
console.log('\n✅ All i18n files are in sync')
|
||||
}
|
||||
}
|
||||
|
||||
async function bootstrap() {
|
||||
if (args.help) {
|
||||
printHelp()
|
||||
return
|
||||
}
|
||||
|
||||
if (args.errors.length) {
|
||||
args.errors.forEach(message => console.error(`❌ ${message}`))
|
||||
printHelp()
|
||||
process.exit(1)
|
||||
return
|
||||
}
|
||||
|
||||
const unknownLangs = targetLangs.filter(lang => !languages.includes(lang))
|
||||
if (unknownLangs.length) {
|
||||
console.error(`❌ Unsupported languages: ${unknownLangs.join(', ')}`)
|
||||
process.exit(1)
|
||||
return
|
||||
}
|
||||
|
||||
await main()
|
||||
}
|
||||
|
||||
bootstrap().catch((error) => {
|
||||
console.error('❌ Unexpected error:', error.message)
|
||||
process.exit(1)
|
||||
})
|
||||
Reference in New Issue
Block a user