diff --git a/app/scripts/translatte/commands/clearServerStrings.ts b/app/scripts/translatte/commands/clearServerStrings.ts index 6df5d923b..d36892240 100644 --- a/app/scripts/translatte/commands/clearServerStrings.ts +++ b/app/scripts/translatte/commands/clearServerStrings.ts @@ -1,11 +1,11 @@ -import { listToGroupList } from "@togglecorp/fujs"; +import { isTruthyString, listToGroupList } from "@togglecorp/fujs"; import { fetchServerState, postLanguageStrings, writeFilePromisify } from "../utils"; async function clearServerStrings(apiUrl: string, authToken: string) { const serverStrings = await fetchServerState(apiUrl, authToken); const bulkActions = listToGroupList( - serverStrings, + serverStrings.filter(({ page_name }) => isTruthyString(page_name)), ({ language }) => language, ({ key, page_name }) => ({ action: "delete" as const, @@ -19,7 +19,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { response: object, }[] = []; - console.log('Pusing delete actions for en...') + console.log('Pushing delete actions for en...') const enResponse = await postLanguageStrings( 'en', bulkActions.en, @@ -31,7 +31,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { logs.push({ responseFor: 'en', response: enResponseJson }); - console.log('Pusing delete actions for fr...') + console.log('Pushing delete actions for fr...') const frResponse = await postLanguageStrings( 'fr', bulkActions.fr, @@ -42,7 +42,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { const frResponseJson = await frResponse.json(); logs.push({ responseFor: 'fr', response: frResponseJson }); - console.log('Pusing delete actions for es...') + console.log('Pushing delete actions for es...') const esResponse = await postLanguageStrings( 'es', bulkActions.es, @@ -52,7 +52,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { const esResponseJson = await esResponse.json(); logs.push({ responseFor: 'es', response: esResponseJson }); - console.log('Pusing delete actions for ar...') + console.log('Pushing delete actions for ar...') const arResponse = await postLanguageStrings( 'ar', bulkActions.ar, diff --git a/app/scripts/translatte/commands/exportServerStringsToExcel.ts b/app/scripts/translatte/commands/exportServerStringsToExcel.ts index bcfa7f8e9..73b8f966f 100644 --- a/app/scripts/translatte/commands/exportServerStringsToExcel.ts +++ b/app/scripts/translatte/commands/exportServerStringsToExcel.ts @@ -1,7 +1,7 @@ import xlsx from 'exceljs'; import { fetchServerState } from "../utils"; -import { isFalsyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { isFalsyString, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; async function exportServerStringsToExcel( apiUrl: string, @@ -35,7 +35,7 @@ async function exportServerStringsToExcel( const keyGroupedStrings = mapToList( listToGroupList( - serverStrings, + serverStrings.filter(({ page_name, key }) => isTruthyString(page_name) && isTruthyString(key)), ({ page_name, key }) => `${page_name}:${key}`, ), (list) => { diff --git a/app/scripts/translatte/commands/pushMigration.ts b/app/scripts/translatte/commands/pushMigration.ts index bce31ef3a..74969d1e9 100644 --- a/app/scripts/translatte/commands/pushMigration.ts +++ b/app/scripts/translatte/commands/pushMigration.ts @@ -102,6 +102,10 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke const serverActionsForCurrentLanguage = actions.flatMap((actionItem) => { if (language === 'en') { if (actionItem.action === 'add') { + if (isFalsyString(actionItem.value)) { + return undefined; + } + return { action: 'set' as const, key: actionItem.key, @@ -168,7 +172,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke ); await writeFilePromisify( - `server-actions.json`, + `/tmp/server-actions.json`, JSON.stringify(serverActions, null, 2), 'utf8', ); @@ -193,7 +197,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke const setActions = actions.filter(({ action }) => action === 'set'); const deleteActions = actions.filter(({ action }) => action === 'delete'); - console.log(`Pusing deleted actions for ${lang}...`) + console.log(`Pushing deleted actions for ${lang}...`) const deleteResponse = await postLanguageStrings( lang, deleteActions, @@ -221,7 +225,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke await applyAction(serverActions.ar.language, serverActions.ar.actions); await writeFilePromisify( - `push-migration-logs.json`, + `/tmp/push-migration-logs.json`, JSON.stringify(logs, null, 2), 'utf8', ); diff --git a/app/scripts/translatte/commands/pushStringsDref.ts b/app/scripts/translatte/commands/pushStringsDref.ts new file mode 100644 index 000000000..3cc25d18e --- /dev/null +++ b/app/scripts/translatte/commands/pushStringsDref.ts @@ -0,0 +1,347 @@ +import xlsx, { CellValue } from 'exceljs'; +import { fetchServerState, getTranslationFileNames, postLanguageStrings, readTranslations, writeFilePromisify } from "../utils"; +import { encodeDate, isDefined, isFalsyString, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { Language, ServerActionItem, SourceStringItem } from '../types'; +import { Md5 } from 'ts-md5'; + + +function getValueFromCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return getValueFromCellValue(cellValue.result); +} + +function getCombinedKey(namespace: string, key: string) { + return `${namespace}:${key}`; +} + +async function createExcel(groupedStrings: Record) { + const workbook = new xlsx.Workbook(); + const now = new Date(); + workbook.created = now; + + const yyyy = now.getFullYear(); + const mm = (now.getMonth() + 1).toString().padStart(2, '0'); + const dd = now.getDate().toString().padStart(2, '0'); + const HH = now.getHours().toString().padStart(2, '0'); + const MM = now.getMinutes().toString().padStart(2, '0'); + + const worksheet = workbook.addWorksheet( + `${yyyy}-${mm}-${dd} ${HH}-${MM}` + ); + + worksheet.columns = [ + { header: 'Namespace', key: 'namespace' }, + { header: 'Key', key: 'key' }, + { header: 'EN', key: 'en' }, + { header: 'FR', key: 'fr' }, + { header: 'ES', key: 'es' }, + { header: 'AR', key: 'ar' }, + ]; + + Object.values(groupedStrings).map((translations) => { + const translationByLang = listToMap( + translations, + ({ language }) => language, + ); + + if (isFalsyString(translationByLang.en)) { + console.info(JSON.stringify(translationByLang, null, 2)); + } else { + worksheet.addRow({ + namespace: translationByLang.en.page_name, + key: translationByLang.en.key, + en: translationByLang.en.value, + fr: translationByLang.fr?.value, + es: translationByLang.es?.value, + ar: translationByLang.ar?.value, + }); + } + }); + + const fileName = `go-dref-updated-strings-${yyyy}-${mm}-${dd}`; + + await workbook.xlsx.writeFile(`${fileName}.xlsx`); +} + +async function pushStringsDref( + projectPath: string, + importFilePath: string, + translationFileNames: string[], + apiUrl: string, + accessToken: string, +) { + const serverState = await fetchServerState(apiUrl); + + const groupedServerStateMapping = listToGroupList( + serverState, + ({ page_name, key }) => getCombinedKey(page_name, key), + ); + + const serverEnStringItems = serverState.filter((string) => string.language === 'en'); + + const translationFiles = await getTranslationFileNames( + projectPath, + Array.isArray(translationFileNames) ? translationFileNames : [translationFileNames], + ); + const { translations } = await readTranslations(translationFiles); + const fileState = translations.map((item) => ({ + ...item, + })); + + const workbook = new xlsx.Workbook(); + await workbook.xlsx.readFile(importFilePath); + + const firstSheet = workbook.worksheets[0]; + const columns = firstSheet.columns.map( + (column) => { + const key = column.values?.[1]?.toString(); + if (isNotDefined(key)) { + return undefined; + } + return { key, column: column.number } + } + ).filter(isDefined); + + const columnMap = listToMap( + columns, + ({ key }) => key, + ({ column }) => column, + ); + + const updatedStrings: SourceStringItem[] = []; + + firstSheet.eachRow((row, i) => { + if (i === 0) { + return; + } + + const enColumnKey = columnMap['EN']; + const frColumnKey = columnMap['FR']; + const esColumnKey = columnMap['ES']; + const arColumnKey = columnMap['AR']; + + const enValue = isDefined(enColumnKey) ? getValueFromCellValue(row.getCell(enColumnKey).value) : undefined; + const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; + const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; + const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + + const serverMatchedStrings = serverEnStringItems.filter(({ value }) => value === enValue); + + serverMatchedStrings.forEach((matchedItem) => { + const combinedKey = getCombinedKey(matchedItem.page_name, matchedItem.key); + + groupedServerStateMapping[combinedKey] = groupedServerStateMapping[combinedKey].map((translationItem) => { + if (translationItem.language === 'fr') { + return { + ...matchedItem, + language: 'fr', + value: String(frValue), + } + } + + if (translationItem.language === 'es') { + return { + ...matchedItem, + language: 'es', + value: String(esValue), + } + } + + if (translationItem.language === 'ar') { + return { + ...matchedItem, + language: 'ar', + value: String(esValue), + } + } + + return translationItem; + }); + + updatedStrings.push({ + ...matchedItem, + language: 'fr', + value: String(frValue), + }); + + updatedStrings.push({ + ...matchedItem, + language: 'es', + value: String(esValue), + }); + + updatedStrings.push({ + ...matchedItem, + language: 'ar', + value: String(arValue), + }); + }); + + const serverMatchedStringsMapping = listToMap( + serverMatchedStrings, + ({ key, page_name }) => getCombinedKey(page_name, key), + () => true, + ); + + const fileMatchedEnStrings = fileState.filter( + ({ value, key, namespace }) => value === enValue && !serverMatchedStringsMapping[getCombinedKey(namespace, key)] + ); + + fileMatchedEnStrings.forEach((matchedItem) => { + const hash = Md5.hashStr(matchedItem.value); + const combinedKey = getCombinedKey(matchedItem.namespace, matchedItem.key); + + groupedServerStateMapping[combinedKey] = [ + { + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'en', + value: matchedItem.value, + }, + { + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'fr', + value: String(frValue), + }, + { + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'es', + value: String(esValue), + }, + { + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'ar', + value: String(arValue), + }, + ]; + + + updatedStrings.push({ + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'en', + value: matchedItem.value, + }); + + updatedStrings.push({ + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'fr', + value: String(frValue), + }); + + updatedStrings.push({ + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'es', + value: String(esValue), + }); + + updatedStrings.push({ + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'ar', + value: String(arValue), + }); + }); + }); + + const languageGroupedActions = mapToList( + listToGroupList( + updatedStrings, + ({ language }) => language, + (languageString) => { + const serverAction: ServerActionItem = { + action: 'set', + key: languageString.key, + page_name: languageString.page_name, + value: languageString.value, + hash: languageString.hash, + } + + return serverAction; + }, + ), + (actions, language) => ({ + language: language as Language, + actions, + }) + ); + + await createExcel(groupedServerStateMapping); + + /* + for (let i = 0; i < languageGroupedActions.length; i++) { + const action = languageGroupedActions[i]; + + console.log(`posting ${action.language} actions...`); + const result = await postLanguageStrings( + action.language, + action.actions, + apiUrl, + accessToken, + ) + + const resultJson = await result.json(); + await writeFilePromisify( + `/tmp/push-${action.language}-strings-dref-logs.json`, + JSON.stringify(resultJson, null, 2), + 'utf8', + ); + } + */ +} + +export default pushStringsDref; diff --git a/app/scripts/translatte/commands/pushStringsFromExcel.ts b/app/scripts/translatte/commands/pushStringsFromExcel.ts index b9da7b38b..5069c9335 100644 --- a/app/scripts/translatte/commands/pushStringsFromExcel.ts +++ b/app/scripts/translatte/commands/pushStringsFromExcel.ts @@ -1,9 +1,55 @@ -import xlsx from 'exceljs'; +import xlsx, { CellValue } from 'exceljs'; import { Md5 } from 'ts-md5'; -import { isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { encodeDate, isDefined, isFalsyString, isNotDefined, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; import { Language, ServerActionItem } from '../types'; -import { postLanguageStrings } from '../utils'; +import { postLanguageStrings, writeFilePromisify } from '../utils'; + +function getValueFromCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return getValueFromCellValue(cellValue.result); +} async function pushStringsFromExcel(importFilePath: string, apiUrl: string, accessToken: string) { const workbook = new xlsx.Workbook(); @@ -11,15 +57,23 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce await workbook.xlsx.readFile(importFilePath); const firstSheet = workbook.worksheets[0]; - const columns = firstSheet.columns.map( - (column) => { - const key = column.values?.[1]?.toString(); - if (isNotDefined(key)) { - return undefined; - } - return { key, column: column.number } + console.info(firstSheet.columnCount); + + const columns: { + key: string; + column: number | undefined; + }[] = []; + + for (let i = 0; i < firstSheet.columnCount; i++) { + const column = firstSheet.columns[i]; + const key = column.values?.[1]?.toString(); + + if (isNotDefined(key)) { + return; } - ).filter(isDefined); + + columns.push({ key, column: column.number }) + } const columnMap = listToMap( columns, @@ -36,28 +90,36 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }[] = []; firstSheet.eachRow( - (row) => { - const keyColumn = columnMap['key']; - const key = isDefined(keyColumn) ? row.getCell(keyColumn).value?.toString() : undefined; + (row, i) => { + if (i === 0) { + return; + } - const namespaceColumn = columnMap['namespace']; - const namespace = isDefined(namespaceColumn) ? row.getCell(namespaceColumn).value?.toString() : undefined; + const keyColumn = columnMap['Key']; + const key = isDefined(keyColumn) ? String(getValueFromCellValue(row.getCell(keyColumn).value)) : undefined; - if (isNotDefined(key) || isNotDefined(namespace)) { + const namespaceColumn = columnMap['Namespace']; + const namespace = isDefined(namespaceColumn) ? String(getValueFromCellValue(row.getCell(namespaceColumn).value)) : undefined; + + if (isFalsyString(key) || isFalsyString(namespace)) { return; } - const enColumn = columnMap['en']; - const en = isDefined(enColumn) ? row.getCell(enColumn).value?.toString() : undefined; + const enColumn = columnMap['EN']; + const en = isDefined(enColumn) ? String(getValueFromCellValue(row.getCell(enColumn).value)) : undefined; - const arColumn = columnMap['ar']; - const ar = isDefined(arColumn) ? row.getCell(arColumn).value?.toString() : undefined; + if (isFalsyString(en)) { + return; + } - const frColumn = columnMap['fr']; - const fr = isDefined(frColumn) ? row.getCell(frColumn).value?.toString() : undefined; + const arColumn = columnMap['AR']; + const ar = isDefined(arColumn) ? String(getValueFromCellValue(row.getCell(arColumn).value)) : undefined; - const esColumn = columnMap['es']; - const es = isDefined(esColumn) ? row.getCell(esColumn).value?.toString() : undefined; + const frColumn = columnMap['FR']; + const fr = isDefined(frColumn) ? String(getValueFromCellValue(row.getCell(frColumn).value)) : undefined; + + const esColumn = columnMap['ES']; + const es = isDefined(esColumn) ? String(getValueFromCellValue(row.getCell(esColumn).value)) : undefined; if (isNotDefined(en)) { return; @@ -73,7 +135,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce hash, }); - if (isDefined(ar)) { + if (isTruthyString(ar)) { strings.push({ key, namespace, @@ -83,7 +145,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }); } - if (isDefined(fr)) { + if (isTruthyString(fr)) { strings.push({ key, namespace, @@ -93,7 +155,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }); } - if (isDefined(es)) { + if (isTruthyString(es)) { strings.push({ key, namespace, @@ -105,6 +167,8 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce } ); + console.info(`Total ${strings.length} actions`); + const languageGroupedActions = mapToList( listToGroupList( strings, @@ -127,16 +191,26 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }) ); - const postPromises = languageGroupedActions.map( - (languageStrings) => postLanguageStrings( - languageStrings.language, - languageStrings.actions, + await writeFilePromisify( + '/tmp/language-grouped-actions.json', + JSON.stringify(languageGroupedActions, null, 2), + 'utf8', + ); + + for (let i = 0; i < languageGroupedActions.length; i++) { + const action = languageGroupedActions[i]; + + console.log(`posting ${action.language} actions...`); + const result = await postLanguageStrings( + action.language, + action.actions, apiUrl, accessToken, - ) - ) + ); - await Promise.all(postPromises); + const resultJson = await result.text(); + console.info(resultJson); + } } export default pushStringsFromExcel; diff --git a/app/scripts/translatte/commands/syncEnStrings.ts b/app/scripts/translatte/commands/syncEnStrings.ts new file mode 100644 index 000000000..3b3b7982e --- /dev/null +++ b/app/scripts/translatte/commands/syncEnStrings.ts @@ -0,0 +1,39 @@ +import { isDefined, isFalsyString } from "@togglecorp/fujs"; +import { fetchServerState, postLanguageStrings, writeFilePromisify } from "../utils"; + +async function syncEnStrings(sourceApiUrl: string, desinationApiUrl: string, authToken: string) { + const serverStrings = await fetchServerState(sourceApiUrl, authToken); + const enStrings = serverStrings.filter((string) => string.language === 'en'); + + const actions = enStrings.map((string) => { + if (isFalsyString(string.key) || isFalsyString(string.page_name) || isFalsyString(string.value)) { + return undefined; + } + + return { + action: 'set' as const, + key: string.key, + page_name: string.page_name, + value: string.value, + hash: string.hash, + }; + }).filter(isDefined); + + console.log("posting en actions..."); + const result = await postLanguageStrings( + 'en', + actions, + desinationApiUrl, + authToken, + ) + + const resultJson = await result.json(); + console.info(resultJson); + await writeFilePromisify( + '/tmp/sync-en-strings-logs.json', + JSON.stringify(resultJson, null, 2), + 'utf8', + ); +} + +export default syncEnStrings; diff --git a/app/scripts/translatte/main.ts b/app/scripts/translatte/main.ts index f8dd65a23..611ff4540 100644 --- a/app/scripts/translatte/main.ts +++ b/app/scripts/translatte/main.ts @@ -14,6 +14,8 @@ import pushMigration from './commands/pushMigration'; import pushStringsFromExcel from './commands/pushStringsFromExcel'; import exportServerStringsToExcel from './commands/exportServerStringsToExcel'; import clearServerStrings from './commands/clearServerStrings'; +import pushStringsDref from './commands/pushStringsDref'; +import syncEnStrings from './commands/syncEnStrings'; const currentDir = cwd(); @@ -254,6 +256,73 @@ yargs(hideBin(process.argv)) ); }, ) + .command( + 'push-strings-dref ', + 'IMPORTANT!!! Temporary command, do not use!', + (yargs) => { + yargs.positional('IMPORT_FILE_PATH', { + type: 'string', + describe: 'Find the import file on IMPORT_FILE_PATH', + }); + yargs.positional('TRANSLATION_FILES', { + type: 'string', + describe: 'Read the files from TRANSLATION_FILES', + }); + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + } + }); + }, + async (argv) => { + const importFilePath = (argv.IMPORT_FILE_PATH as string); + + await pushStringsDref( + currentDir, + importFilePath, + argv.TRANSLATION_FILES as string[], + argv.apiUrl as string, + argv.authToken as string, + ); + }, + ) + .command( + 'sync-en-strings', + 'IMPORTANT!!! Temporary command, do not use!', + (yargs) => { + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'source-api-url': { + type: 'string', + describe: 'URL for the source API server', + require: true, + }, + 'destination-api-url': { + type: 'string', + describe: 'URL for the destination', + require: true, + } + }); + }, + async (argv) => { + await syncEnStrings( + argv.sourceApiUrl as string, + argv.destinationApiUrl as string, + argv.authToken as string, + ); + }, + ) .command( 'export-server-strings ', 'Export server strings to excel file', diff --git a/go-api b/go-api index 596b1f2c2..42587133d 160000 --- a/go-api +++ b/go-api @@ -1 +1 @@ -Subproject commit 596b1f2c24014148879a97cc069a62205dfdc24d +Subproject commit 42587133db4ab72094602b69adb0e6e96b970097