Merge branch 'main' into HEAD

tools/scripts/package.json

@@ -13,9 +13,11 @@
   },
   "main": "src/index.ts",
   "scripts": {
-    "build": "tsc",
+    "build": "tsc --project tsconfig.build.json",
     "build-template-with-local-pkgs": "pnpm runts src/build-template-with-local-pkgs.ts",
     "gen-templates": "pnpm runts src/generate-template-variations.ts",
+    "generateTranslations:core": "node --no-deprecation --import @swc-node/register/esm-register src/generateTranslations/core.ts",
+    "generateTranslations:plugin-multi-tenant": "node --no-deprecation --import @swc-node/register/esm-register src/generateTranslations/plugin-multi-tenant.ts",
     "license-check": "pnpm runts src/license-check.ts",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
@@ -32,6 +34,12 @@
     "create-payload-app": "workspace:*",
     "csv-stringify": "^6.5.2",
     "license-checker": "25.0.1",
-    "open": "^10.1.0"
+    "open": "^10.1.0",
+    "payload": "workspace:*"
+  },
+  "devDependencies": {
+    "@payloadcms/plugin-multi-tenant": "workspace:*",
+    "@payloadcms/richtext-lexical": "workspace:*",
+    "@payloadcms/translations": "workspace:*"
   }
 }

tools/scripts/src/generate-template-variations.ts

@@ -152,6 +152,7 @@ async function main() {
       generateLockfile: true,
       sharp: true,
       skipConfig: true, // Do not copy the payload.config.ts file from the base template
+      skipReadme: true, // Do not copy the README.md file from the base template
       storage: 'localDisk',
       // The blank template is used as a base for create-payload-app functionality,
       // so we do not configure the payload.config.ts file, which leaves the placeholder comments.
@@ -262,10 +263,10 @@ async function main() {
 
     if (generateLockfile) {
       log('Generating pnpm-lock.yaml')
-      execSyncSafe(`pnpm install --ignore-workspace`, { cwd: destDir })
+      execSyncSafe(`pnpm install --ignore-workspace --no-frozen-lockfile`, { cwd: destDir })
     } else {
       log('Installing dependencies without generating lockfile')
-      execSyncSafe(`pnpm install --ignore-workspace`, { cwd: destDir })
+      execSyncSafe(`pnpm install --ignore-workspace --no-frozen-lockfile`, { cwd: destDir })
       await fs.rm(`${destDir}/pnpm-lock.yaml`, { force: true })
     }
 

tools/scripts/src/generateTranslations/core.ts

@@ -0,0 +1,31 @@
+import type { AcceptedLanguages, GenericTranslationsObject } from '@payloadcms/translations'
+
+import { translations } from '@payloadcms/translations/all'
+import { enTranslations } from '@payloadcms/translations/languages/en'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+import { translateObject } from './utils/index.js'
+
+const filename = fileURLToPath(import.meta.url)
+const dirname = path.dirname(filename)
+
+const allTranslations: {
+  [key in AcceptedLanguages]?: {
+    dateFNSKey: string
+    translations: GenericTranslationsObject
+  }
+} = {}
+
+for (const key of Object.keys(translations) as AcceptedLanguages[]) {
+  allTranslations[key] = {
+    dateFNSKey: translations[key]?.dateFNSKey || 'unknown-date-fns-key',
+    translations: translations[key]?.translations || {},
+  }
+}
+
+void translateObject({
+  allTranslationsObject: allTranslations,
+  fromTranslationsObject: enTranslations,
+  targetFolder: path.resolve(dirname, '../../../../packages/translations/src/languages'),
+})

tools/scripts/src/generateTranslations/plugin-multi-tenant.ts

@@ -0,0 +1,39 @@
+import type { AcceptedLanguages, GenericTranslationsObject } from '@payloadcms/translations'
+
+import { translations } from '@payloadcms/plugin-multi-tenant/translations/languages/all'
+import { enTranslations } from '@payloadcms/plugin-multi-tenant/translations/languages/en'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+import { translateObject } from './utils/index.js'
+
+const filename = fileURLToPath(import.meta.url)
+const dirname = path.dirname(filename)
+
+const allTranslations: {
+  [key in AcceptedLanguages]?: {
+    dateFNSKey: string
+    translations: GenericTranslationsObject
+  }
+} = {}
+
+for (const key of Object.keys(translations)) {
+  allTranslations[key as AcceptedLanguages] = {
+    dateFNSKey: translations[key as AcceptedLanguages]?.dateFNSKey ?? 'unknown-date-fns-key',
+    translations: translations[key as AcceptedLanguages]?.translations ?? {},
+  }
+}
+
+void translateObject({
+  allTranslationsObject: allTranslations,
+  fromTranslationsObject: enTranslations,
+  targetFolder: path.resolve(
+    dirname,
+    '../../../../packages/plugin-multi-tenant/src/translations/languages',
+  ),
+  tsFilePrefix: `import type { PluginDefaultTranslationsObject, PluginLanguage } from '../types.js'\n\nexport const {{locale}}Translations: PluginDefaultTranslationsObject = `,
+  tsFileSuffix: `\n\nexport const {{locale}}: PluginLanguage = {
+    dateFNSKey: {{dateFNSKey}},
+    translations: {{locale}}Translations,
+  }  `,
+})

tools/scripts/src/generateTranslations/richtext-lexical.ts

@@ -0,0 +1,75 @@
+import type {
+  AcceptedLanguages,
+  GenericLanguages,
+  GenericTranslationsObject,
+} from '@payloadcms/translations'
+
+import * as fs from 'node:fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+import { translateObject } from './utils/index.js'
+const filename = fileURLToPath(import.meta.url)
+const dirname = path.dirname(filename)
+
+// Function to get all files with a specific name recursively in all subdirectories
+function findFilesRecursively(startPath: string, filter: string): string[] {
+  let results: string[] = []
+
+  const entries = fs.readdirSync(startPath, { withFileTypes: true })
+
+  entries.forEach((dirent) => {
+    const fullPath = path.join(startPath, dirent.name)
+
+    if (dirent.isDirectory()) {
+      results = results.concat(findFilesRecursively(fullPath, filter))
+    } else {
+      if (dirent.name === filter) {
+        results.push(fullPath)
+      }
+    }
+  })
+
+  return results
+}
+
+const i18nFilePaths = findFilesRecursively(
+  path.resolve(dirname, '../../../../packages/richtext-lexical/src'),
+  'i18n.ts',
+)
+
+async function translate() {
+  for (const i18nFilePath of i18nFilePaths) {
+    const imported = await import(i18nFilePath)
+    const translationsObject = imported.i18n as Partial<GenericLanguages>
+    const allTranslations: {
+      [key in AcceptedLanguages]?: {
+        dateFNSKey: string
+        translations: GenericTranslationsObject
+      }
+    } = {}
+    for (const lang in translationsObject) {
+      allTranslations[lang as AcceptedLanguages] = {
+        dateFNSKey: 'en',
+        translations: translationsObject?.[lang as keyof GenericLanguages] || {},
+      }
+    }
+
+    if (translationsObject.en) {
+      console.log('Translating', i18nFilePath)
+      await translateObject({
+        allTranslationsObject: allTranslations,
+        fromTranslationsObject: translationsObject.en,
+        inlineFile: i18nFilePath,
+        tsFilePrefix: `import { GenericLanguages } from '@payloadcms/translations'
+  
+  export const i18n: Partial<GenericLanguages> = `,
+        tsFileSuffix: ``,
+      })
+    } else {
+      console.error(`No English translations found in ${i18nFilePath}`)
+    }
+  }
+}
+
+void translate()

tools/scripts/src/generateTranslations/utils/applyEslintFixes.ts

@@ -0,0 +1,8 @@
+import { ESLint } from 'eslint'
+
+export async function applyEslintFixes(text: string, filePath: string): Promise<string> {
+  const eslint = new ESLint({ fix: true })
+  const results = await eslint.lintText(text, { filePath })
+  const result = results[0] || { output: text }
+  return result.output || text // Return the fixed content or the original if no fixes were made.
+}

tools/scripts/src/generateTranslations/utils/findMissingKeys.ts

@@ -0,0 +1,30 @@
+import type { GenericTranslationsObject } from '@payloadcms/translations'
+
+/**
+ * Returns keys which are present in baseObj but not in targetObj
+ */
+export function findMissingKeys(
+  baseObj: GenericTranslationsObject,
+  targetObj: GenericTranslationsObject,
+  prefix = '',
+): string[] {
+  let missingKeys: string[] = []
+
+  for (const key in baseObj) {
+    const baseValue = baseObj[key]
+    const targetValue = targetObj[key]
+    if (typeof baseValue === 'object') {
+      missingKeys = missingKeys.concat(
+        findMissingKeys(
+          baseValue,
+          typeof targetValue === 'object' && targetValue ? targetValue : {},
+          `${prefix}${key}.`,
+        ),
+      )
+    } else if (!(key in targetObj)) {
+      missingKeys.push(`${prefix}${key}`)
+    }
+  }
+
+  return missingKeys
+}

tools/scripts/src/generateTranslations/utils/generateTsObjectLiteral.ts

@@ -0,0 +1,15 @@
+import type { JsonObject } from 'payload'
+
+export function generateTsObjectLiteral(obj: JsonObject): string {
+  const lines: string[] = []
+  const entries = Object.entries(obj)
+  for (const [key, value] of entries) {
+    const safeKey = /^[\w$]+$/.test(key) ? key : JSON.stringify(key)
+    const line =
+      typeof value === 'object' && value !== null
+        ? `${safeKey}: ${generateTsObjectLiteral(value)}`
+        : `${safeKey}: ${JSON.stringify(value)}`
+    lines.push(line)
+  }
+  return `{\n  ${lines.join(',\n  ')}\n}`
+}

tools/scripts/src/generateTranslations/utils/index.ts

@@ -0,0 +1,257 @@
+/* eslint no-console: 0 */
+
+import type {
+  AcceptedLanguages,
+  GenericLanguages,
+  GenericTranslationsObject,
+} from '@payloadcms/translations'
+
+import { acceptedLanguages } from '@payloadcms/translations'
+import fs from 'fs'
+import path from 'path'
+import { deepMergeSimple } from 'payload/shared'
+import { format } from 'prettier'
+
+import { applyEslintFixes } from './applyEslintFixes.js'
+import { findMissingKeys } from './findMissingKeys.js'
+import { generateTsObjectLiteral } from './generateTsObjectLiteral.js'
+import { sortKeys } from './sortKeys.js'
+import { translateText } from './translateText.js'
+
+/**
+ *
+ * props.allTranslationsObject:
+ * @Example
+ * ```ts
+ * {
+ *   en: {
+ *     lexical: {
+ *       link: {
+ *         editLink: 'Edit link',
+ *         invalidURL: 'Invalid URL',
+ *         removeLink: 'Remove link',
+ *       },
+ *     },
+ *   },
+ *   de: {
+ *     lexical: {
+ *       // ...
+ *     }
+ *   },
+ *   // ...
+ * }
+ *```
+ *
+ * @param props
+ */
+export async function translateObject(props: {
+  allTranslationsObject: {
+    [key in AcceptedLanguages]?: {
+      dateFNSKey: string
+      translations: GenericTranslationsObject
+    }
+  }
+  fromTranslationsObject: GenericTranslationsObject
+  /**
+   *
+   * If set, will output the entire translations object (incl. all locales) to this file.
+   *
+   * @default false
+   */
+  inlineFile?: string
+  languages?: AcceptedLanguages[]
+  targetFolder?: string
+  tsFilePrefix?: string
+  tsFileSuffix?: string
+}) {
+  const {
+    allTranslationsObject,
+    fromTranslationsObject,
+    inlineFile,
+    languages = acceptedLanguages.filter((lang) => lang !== 'en'),
+    targetFolder = '',
+    tsFilePrefix = `import type { DefaultTranslationsObject, Language } from '../types.js'\n\nexport const {{locale}}Translations: DefaultTranslationsObject = `,
+    tsFileSuffix = `\n\nexport const {{locale}}: Language = {
+  dateFNSKey: {{dateFNSKey}},
+  translations: {{locale}}Translations,
+}  `,
+  } = props
+
+  const allTranslatedTranslationsObject: {
+    [key in AcceptedLanguages]?: {
+      dateFNSKey: string
+      translations: GenericTranslationsObject
+    }
+  } = JSON.parse(JSON.stringify(allTranslationsObject))
+  const allOnlyNewTranslatedTranslationsObject: GenericLanguages = {}
+
+  const translationPromises: Promise<void>[] = []
+
+  for (const targetLang of languages) {
+    if (!allTranslatedTranslationsObject?.[targetLang]) {
+      allTranslatedTranslationsObject[targetLang] = {
+        dateFNSKey: targetLang,
+        translations: {},
+      }
+    }
+    const keysWhichDoNotExistInFromlang = findMissingKeys(
+      allTranslatedTranslationsObject?.[targetLang].translations,
+      fromTranslationsObject,
+    )
+    if (keysWhichDoNotExistInFromlang.length) {
+      console.log(`Keys which do not exist in English:`, keysWhichDoNotExistInFromlang)
+    }
+
+    /**
+     * If a key does not exist in the fromTranslationsObject, it should be deleted from the target language object
+     */
+    for (const key of keysWhichDoNotExistInFromlang) {
+      // Delete those keys in the target language object obj[lang]
+      const keys: string[] = key.split('.')
+      let targetObj = allTranslatedTranslationsObject?.[targetLang].translations
+      for (let i = 0; i < keys.length - 1; i += 1) {
+        const nextObj = targetObj[keys[i] as string]
+        if (typeof nextObj !== 'object') {
+          throw new Error(`Key ${keys[i]} is not an object in ${targetLang} (1)`)
+        }
+        targetObj = nextObj
+      }
+      delete targetObj[keys[keys.length - 1] as string]
+    }
+
+    if (!allTranslatedTranslationsObject?.[targetLang].translations) {
+      allTranslatedTranslationsObject[targetLang].translations = {}
+    }
+    const missingKeys = findMissingKeys(
+      fromTranslationsObject,
+      allTranslatedTranslationsObject?.[targetLang].translations,
+    )
+
+    if (missingKeys.length) {
+      console.log('Missing keys for lang', targetLang, ':', missingKeys)
+    }
+
+    for (const missingKey of missingKeys) {
+      const keys: string[] = missingKey.split('.')
+      const sourceText = keys.reduce((acc, key) => acc[key], fromTranslationsObject)
+      if (!sourceText || typeof sourceText !== 'string') {
+        throw new Error(
+          `Missing key ${missingKey} or key not "leaf" in fromTranslationsObject for lang ${targetLang}. (2)`,
+        )
+      }
+
+      if (translationPromises.length >= 12) {
+        // Wait for one of the promises to resolve before adding a new one
+        await Promise.race(translationPromises)
+      }
+
+      translationPromises.push(
+        translateText(sourceText, targetLang).then((translated) => {
+          if (!allOnlyNewTranslatedTranslationsObject[targetLang]) {
+            allOnlyNewTranslatedTranslationsObject[targetLang] = {}
+          }
+          let targetObj = allOnlyNewTranslatedTranslationsObject?.[targetLang]
+          for (let i = 0; i < keys.length - 1; i += 1) {
+            if (!targetObj[keys[i] as string]) {
+              targetObj[keys[i] as string] = {}
+            }
+            const nextObj = targetObj[keys[i] as string]
+            if (typeof nextObj !== 'object') {
+              throw new Error(`Key ${keys[i]} is not an object in ${targetLang} (3)`)
+            }
+            targetObj = nextObj
+          }
+          targetObj[keys[keys.length - 1] as string] = translated
+
+          allTranslatedTranslationsObject[targetLang]!.translations = sortKeys(
+            deepMergeSimple(
+              allTranslatedTranslationsObject[targetLang]!.translations,
+              allOnlyNewTranslatedTranslationsObject[targetLang],
+            ),
+          )
+        }),
+      )
+    }
+  }
+
+  //await Promise.all(translationPromises)
+  for (const promise of translationPromises) {
+    await promise
+  }
+
+  // merge with existing translations
+  // console.log('Merged object:', allTranslatedTranslationsObject)
+
+  console.log('New translations:', allOnlyNewTranslatedTranslationsObject)
+
+  if (inlineFile?.length) {
+    const simpleTranslationsObject: GenericTranslationsObject = {}
+    for (const lang in allTranslatedTranslationsObject) {
+      if (lang in allTranslatedTranslationsObject) {
+        simpleTranslationsObject[lang as keyof typeof allTranslatedTranslationsObject] =
+          allTranslatedTranslationsObject[
+            lang as keyof typeof allTranslatedTranslationsObject
+          ]!.translations
+      }
+    }
+
+    // write allTranslatedTranslationsObject
+    const filePath = path.resolve(inlineFile)
+    let fileContent: string = `${tsFilePrefix}${generateTsObjectLiteral(simpleTranslationsObject)}\n`
+
+    // suffix
+    fileContent += `${tsFileSuffix}\n`
+
+    // eslint
+    fileContent = await applyEslintFixes(fileContent, filePath)
+
+    // prettier
+    fileContent = await format(fileContent, {
+      parser: 'typescript',
+      printWidth: 100,
+      semi: false,
+      singleQuote: true,
+      trailingComma: 'all',
+    })
+
+    fs.writeFileSync(filePath, fileContent, 'utf8')
+  } else {
+    // save
+
+    for (const key of languages) {
+      if (!allTranslatedTranslationsObject?.[key]) {
+        return
+      }
+
+      // e.g. sanitize rs-latin to rsLatin
+      const sanitizedKey = key.replace(
+        /-(\w)(\w*)/g,
+        (_, firstLetter, remainingLetters) =>
+          firstLetter.toUpperCase() + remainingLetters.toLowerCase(),
+      )
+      const filePath = path.resolve(targetFolder, `${sanitizedKey}.ts`)
+
+      // prefix & translations
+      let fileContent: string = `${tsFilePrefix.replace('{{locale}}', sanitizedKey)}${generateTsObjectLiteral(allTranslatedTranslationsObject[key]?.translations || {})}\n`
+
+      // suffix
+      fileContent += `${tsFileSuffix.replaceAll('{{locale}}', sanitizedKey).replaceAll('{{dateFNSKey}}', `'${allTranslatedTranslationsObject[key]?.dateFNSKey}'`)}\n`
+
+      // eslint
+      fileContent = await applyEslintFixes(fileContent, filePath)
+
+      // prettier
+      fileContent = await format(fileContent, {
+        parser: 'typescript',
+        printWidth: 100,
+        semi: false,
+        singleQuote: true,
+        trailingComma: 'all',
+      })
+
+      fs.writeFileSync(filePath, fileContent, 'utf8')
+    }
+  }
+
+  return allTranslatedTranslationsObject
+}

tools/scripts/src/generateTranslations/utils/sortKeys.ts

@@ -0,0 +1,18 @@
+export function sortKeys(obj: any): any {
+  if (typeof obj !== 'object' || obj === null) {
+    return obj
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map(sortKeys)
+  }
+
+  const sortedKeys = Object.keys(obj).sort()
+  const sortedObj: { [key: string]: any } = {}
+
+  for (const key of sortedKeys) {
+    sortedObj[key] = sortKeys(obj[key])
+  }
+
+  return sortedObj
+}

tools/scripts/src/generateTranslations/utils/translateText.ts

@@ -0,0 +1,48 @@
+import dotenv from 'dotenv'
+import { fileURLToPath } from 'node:url'
+import path from 'path'
+const filename = fileURLToPath(import.meta.url)
+const dirname = path.dirname(filename)
+
+dotenv.config({ path: path.resolve(dirname, '../../../../', '.env') })
+
+export async function translateText(text: string, targetLang: string) {
+  const response = await fetch('https://api.openai.com/v1/chat/completions', {
+    body: JSON.stringify({
+      max_tokens: 150,
+      messages: [
+        {
+          content: `Only respond with the translation of the text you receive. The original language is English and the translation language is ${targetLang}. Use formal and professional language. Only respond with the translation - do not say anything else. If you cannot translate the text, respond with "[SKIPPED]". Do not translate text inside double curly braces, i.e. "{{do_not_translate}}".`,
+          role: 'system',
+        },
+        {
+          content: text,
+          role: 'user',
+        },
+      ],
+      model: 'gpt-4',
+    }),
+    headers: {
+      Authorization: `Bearer ${process.env.OPENAI_KEY}`,
+      'Content-Type': 'application/json',
+    },
+    method: 'POST',
+  })
+  try {
+    const data = await response.json()
+
+    if (response.status > 200) {
+      console.log(data.error)
+    } else {
+      if (data?.choices?.[0]) {
+        console.log('  Old text:', text, 'New text:', data.choices[0].message.content.trim())
+        return data.choices[0].message.content.trim()
+      } else {
+        console.log(`Could not translate: ${text} in lang: ${targetLang}`)
+      }
+    }
+  } catch (e) {
+    console.error('Error translating:', text, 'to', targetLang, 'response', response, '. Error:', e)
+    throw e
+  }
+}

tools/scripts/tsconfig.build.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "strict": true
+  },
+  "references": [
+    { "path": "../../packages/translations" },
+    { "path": "../../packages/richtext-lexical" },
+    { "path": "../../packages/plugin-multi-tenant" }
+  ],
+  "exclude": ["./src/generateTranslations"]
+}

tools/scripts/tsconfig.json

@@ -2,5 +2,6 @@
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
     "strict": true,
-  }
+  },
+  "references": [{ "path": "../../packages/translations" }, { "path": "../../packages/richtext-lexical"}, { "path": "../../packages/plugin-multi-tenant"}]
 }