accounts-frontend/packages/scripts/i18n-crowdin.ts

/* eslint-env node */
/* eslint-disable */

import chalk from 'chalk';
import fs from 'fs';
import path from 'path';

import axios from 'axios';
import JSON5 from 'json5';
import Crowdin, { SourceFilesModel } from '@crowdin/crowdin-api-client';
import ProgressBar from 'progress';
import ch from 'chalk';
import iso639 from 'iso-639-1';
import { prompt, DistinctQuestion } from 'inquirer';

import getRepoInfo from 'git-repo-info';

import { crowdin as config } from './../../config';

if (!config.apiKey) {
    console.error(ch.red`crowdinApiKey is required`);
    process.exit(126);
}

const PROJECT_ID = config.projectId;
const CROWDIN_FILE_PATH = config.filePath;
const SOURCE_LANG = config.sourceLang;
const LANG_DIR = config.basePath;
const INDEX_FILE_NAME = 'index.js';
const MIN_RELEASE_PROGRESS = config.minApproved;

const crowdin = new Crowdin({
    token: config.apiKey,
});

/**
 * Locales that has been verified by core team members
 */
const releasedLocales: ReadonlyArray<string> = ['be', 'fr', 'id', 'pt', 'ru', 'uk', 'vi', 'zh'];

/**
 * Map Crowdin locales into our internal locales representation
 */
const LOCALES_MAP: Record<string, string> = {
    'pt-BR': 'pt',
    'zh-CN': 'zh',
};

/**
 * This array allows us to customise native languages names,
 * because ISO-639-1 sometimes is strange
 */
const NATIVE_NAMES_MAP: Record<string, string> = {
    be: 'Беларуская',
    id: 'Bahasa Indonesia',
    lt: 'Lietuvių',
    pl: 'Polski',
    pt: 'Português do Brasil',
    sr: 'Српски',
    ro: 'Română',
    zh: '简体中文',
};

/**
 * This arrays allows us to override Crowdin English languages names
 */
const ENGLISH_NAMES_MAP: Record<string, string> = {
    pt: 'Portuguese, Brazilian',
    sr: 'Serbian',
    zh: 'Simplified Chinese',
};

/**
 * Converts Crowdin's language code to our internal value
 */
function toInternalLocale(code: string): string {
    return LOCALES_MAP[code] || code;
}

/**
 * Форматирует входящий объект с переводами в итоговую строку в том формате, в каком они
 * хранятся в самом приложении
 */
function serializeToModule(translates: Record<string, any>): string {
    const src = JSON5.stringify(sortByKeys(translates), null, 4);

    return `export default ${src};\n`;
}

// http://stackoverflow.com/a/29622653/5184751
function sortByKeys<T extends Record<string, any>>(object: T): T {
    return Object.keys(object)
        .sort()
        .reduce((result, key) => {
            // @ts-ignore
            result[key] = object[key];

            return result;
        }, {} as T);
}

interface IndexFileEntry {
    code: string;
    name: string;
    englishName: string;
    progress: number;
    isReleased: boolean;
}

function getLocaleFilePath(languageId: string): string {
    return path.join(LANG_DIR, `${toInternalLocale(languageId)}.json`);
}

async function findDirectoryId(path: string, branchId?: number): Promise<number|undefined> {
    const { data: dirsResponse } = await crowdin.sourceFilesApi.listProjectDirectories(PROJECT_ID, branchId);
    const dirs = dirsResponse.map((dirData) => dirData.data);

    const result = path.split('/').reduce((parentDir, dirName) => {
        // directoryId is nullable when a directory has no parent
        return dirs.find((dir) => dir.directoryId === parentDir && dir.name === dirName)?.id;
    }, null as number|null|undefined);

    return result || undefined;
}

async function findFileId(filePath: string, branchId?: number): Promise<number|undefined> {
    const fileName = path.basename(filePath);
    const dirPath = path.dirname(filePath);
    let directoryId: number|null = null;
    if (dirPath !== '') {
        directoryId = await findDirectoryId(dirPath, branchId) || null;
    }

    // We're receiving files list without branch filter until https://github.com/crowdin/crowdin-api-client-js/issues/63
    // will be resolved. But right now it doesn't matter because for each branch directories will have its own ids,
    // so if the file is stored into the some directory, algorithm will find correct file.
    const { data: filesResponse } = await crowdin.sourceFilesApi.listProjectFiles(PROJECT_ID/*, branchId*/);
    const files = filesResponse.map((fileData) => fileData.data);

    return files.find((file) => file.directoryId === directoryId && file.name === fileName)?.id;
}

async function findBranchId(branchName: string): Promise<number|undefined> {
    const { data: branchesList } = await crowdin.sourceFilesApi.listProjectBranches(PROJECT_ID, branchName);
    const branch = branchesList.find(({ data: branch }) => branch.name === branchName);

    return branch?.data.id;
}

async function ensureDirectory(dirPath: string, branchId?: number): Promise<number> {
    const { data: dirsResponse } = await crowdin.sourceFilesApi.listProjectDirectories(PROJECT_ID, branchId);
    const dirs = dirsResponse.map((dirData) => dirData.data);

    return dirPath.split('/').reduce(async (parentDirPromise, name) => {
        const parentDir = await parentDirPromise;
        const directoryId = dirs.find((dir) => dir.directoryId === parentDir && dir.name === name)?.id;
        if (directoryId) {
            return directoryId;
        }

        const createDirRequest: SourceFilesModel.CreateDirectoryRequest = { name };
        if (directoryId) {
            createDirRequest['directoryId'] = directoryId;
        } else if (branchId) {
            createDirRequest['branchId'] = branchId;
        }

        const dirResponse = await crowdin.sourceFilesApi.createDirectory(PROJECT_ID, createDirRequest);

        return dirResponse.data.id;
        // @ts-ignore
    }, Promise.resolve<number>(null));
}

async function pull(): Promise<void> {
    const { branch: branchName } = getRepoInfo();
    const isMasterBranch = branchName === 'master';
    let branchId: number|undefined;
    if (!isMasterBranch) {
        console.log(`Current branch isn't ${chalk.green('master')}, will try to pull translates from the ${chalk.green(branchName)} branch`);
        branchId = await findBranchId(branchName);
        if (!branchId) {
            console.log(`Branch ${chalk.green(branchName)} isn't found, will use ${chalk.green('master')} instead`);
        }
    }

    console.log('Loading file info...');
    const fileId = await findFileId(CROWDIN_FILE_PATH, branchId);
    if (!fileId) {
        throw new Error('Cannot find the file');
    }

    console.log('Pulling translation progress...');
    const { data: translationProgress } = await crowdin.translationStatusApi.getFileProgress(PROJECT_ID, fileId, 100);

    const localesToPull: Array<string> = [];
    const indexFileEntries: Record<string, IndexFileEntry> = {
        en: {
            code: 'en',
            name: 'English',
            englishName: 'English',
            progress: 100,
            isReleased: true,
        },
    };

    translationProgress.forEach(({ data: { languageId, approvalProgress } }) => {
        const locale = toInternalLocale(languageId);
        if (releasedLocales.includes(locale) || approvalProgress >= MIN_RELEASE_PROGRESS) {
            localesToPull.push(languageId);
            indexFileEntries[locale] = {
                code: locale,
                name: NATIVE_NAMES_MAP[locale] || iso639.getNativeName(locale),
                englishName: ENGLISH_NAMES_MAP[locale] || iso639.getName(locale),
                progress: approvalProgress,
                isReleased: releasedLocales.includes(locale),
            };
        }
    });

    // Add prefix 'c' to current and total to prevent filling thees placeholders with real values
    const downloadingProgressBar = new ProgressBar('Downloading translates :bar :percent | :cCurrent/:total', {
        total: localesToPull.length,
        incomplete: '\u2591',
        complete: '\u2588',
        width: Object.keys(indexFileEntries).length - 1,
    });
    let downloadingReady = 0;

    const promises = localesToPull.map(async (languageId): Promise<void> => {
        const { data: { url } } = await crowdin.translationsApi.buildProjectFileTranslation(PROJECT_ID, fileId, {
            targetLanguageId: languageId,
            exportApprovedOnly: true,
        });

        const { data: fileContents } = await axios.get(url, {
            // Disable response parsing
            transformResponse: [],
        });
        fs.writeFileSync(getLocaleFilePath(languageId), fileContents);

        downloadingProgressBar.update(++downloadingReady / localesToPull.length, {
            cCurrent: downloadingReady,
        });
    });

    await Promise.all(promises);

    console.log('Writing an index file');

    fs.writeFileSync(path.join(LANG_DIR, INDEX_FILE_NAME), serializeToModule(indexFileEntries));

    console.log(ch.green('The index file was successfully written'));
}

async function push(): Promise<void> {
    if (!fs.existsSync(getLocaleFilePath(SOURCE_LANG))) {
        console.error(chalk.red(`File for the source language doesn't exists. Run ${chalk.green('yarn i18n:extract')} to generate the source language file.`));
        return;
    }

    const questions: Array<DistinctQuestion> = [{
        name: 'disapproveTranslates',
        type: 'confirm',
        default: true,
        message: 'Disapprove changed lines?',
    }];

    const { branch: branchName } = getRepoInfo();
    const isMasterBranch = branchName === 'master';
    if (!isMasterBranch) {
        questions.push({
            name: 'publishInBranch',
            type: 'confirm',
            default: true,
            message: `Should be strings published in its own branch [${chalk.green(getRepoInfo().branch)}]?`,
        });
    }

    const { disapproveTranslates, publishInBranch = false } = await prompt(questions);

    let branchId: number|undefined;
    if (publishInBranch) {
        console.log('Loading the branch info...');
        branchId = await findBranchId(branchName);
        if (!branchId) {
            console.log('Branch doesn\'t exists. Creating...');
            const { data: branchResponse } = await crowdin.sourceFilesApi.createBranch(PROJECT_ID, {
                name: branchName,
            });
            branchId = branchResponse.id;
        }
    }

    console.log("Loading the file info...");
    const fileId = await findFileId(CROWDIN_FILE_PATH, branchId);
    let dirId: number|undefined;
    if (!fileId) {
        const dirPath = path.dirname(CROWDIN_FILE_PATH);
        if (dirPath !== '') {
            console.log("Ensuring necessary directories structure...");
            dirId = await ensureDirectory(dirPath, branchId);
        }
    }

    console.log('Uploading the source file to the storage...');
    const { data: storageResponse } = await crowdin.uploadStorageApi.addStorage(
        path.basename(CROWDIN_FILE_PATH),
        fs.readFileSync(getLocaleFilePath(SOURCE_LANG)),
    );

    if (fileId) {
        console.log(`Applying the new revision...`);
        await crowdin.sourceFilesApi.updateOrRestoreFile(PROJECT_ID, fileId, {
            storageId: storageResponse.id,
            updateOption: disapproveTranslates
                ? SourceFilesModel.UpdateOption.CLEAR_TRANSLATIONS_AND_APPROVALS
                : SourceFilesModel.UpdateOption.KEEP_TRANSLATIONS_AND_APPROVALS,
        });
    } else {
        console.log(`Uploading the file...`);
        const createFileRequest: SourceFilesModel.CreateFileRequest = {
            storageId: storageResponse.id,
            name: path.basename(CROWDIN_FILE_PATH),
        };

        if (dirId) {
            createFileRequest['directoryId'] = dirId;
        } else if (branchId) {
            createFileRequest['branchId'] = branchId;
        }

        await crowdin.sourceFilesApi.createFile(PROJECT_ID, createFileRequest);
    }

    console.log(ch.green('Success'));
}

try {
    const action = process.argv[2];

    switch (action) {
        case 'pull':
            pull();
            break;
        case 'push':
            push();
            break;
        default:
            console.error(`Unknown action ${action}`);
    }
} catch (exception) {
    console.error(exception);
}