From 4b7971f60fd7ab7cbbd6fd50b14e9361fe6de036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Wed, 2 Jan 2019 01:41:07 +0100 Subject: [PATCH] NOISSUE hotloading of translations and use of local PO files The hotloading is still inefficient --- api/logic/CMakeLists.txt | 2 + api/logic/translations/POTranslator.cpp | 373 +++++++++++++++++++ api/logic/translations/POTranslator.h | 16 + api/logic/translations/TranslationsModel.cpp | 306 ++++++++++++--- api/logic/translations/TranslationsModel.h | 4 +- 5 files changed, 650 insertions(+), 51 deletions(-) create mode 100644 api/logic/translations/POTranslator.cpp create mode 100644 api/logic/translations/POTranslator.h diff --git a/api/logic/CMakeLists.txt b/api/logic/CMakeLists.txt index b8e45b30..45b01b9f 100644 --- a/api/logic/CMakeLists.txt +++ b/api/logic/CMakeLists.txt @@ -387,6 +387,8 @@ add_unit_test(JavaVersion set(TRANSLATIONS_SOURCES translations/TranslationsModel.h translations/TranslationsModel.cpp + translations/POTranslator.h + translations/POTranslator.cpp ) set(TOOLS_SOURCES diff --git a/api/logic/translations/POTranslator.cpp b/api/logic/translations/POTranslator.cpp new file mode 100644 index 00000000..1ffcb9a4 --- /dev/null +++ b/api/logic/translations/POTranslator.cpp @@ -0,0 +1,373 @@ +#include "POTranslator.h" + +#include +#include "FileSystem.h" + +struct POEntry +{ + QString text; + bool fuzzy; +}; + +struct POTranslatorPrivate +{ + QString filename; + QHash mapping; + QHash mapping_disambiguatrion; + bool loaded = false; + + void reload(); +}; + +class ParserArray : public QByteArray +{ +public: + ParserArray(const QByteArray &in) : QByteArray(in) + { + } + bool chomp(const char * data, int length) + { + if(startsWith(data)) + { + remove(0, length); + return true; + } + return false; + } + bool chompString(QByteArray & appendHere) + { + QByteArray msg; + bool escape = false; + if(size() < 2) + { + qDebug() << "String fragment is too short"; + return false; + } + if(!startsWith('"')) + { + qDebug() << "String fragment does not start with \""; + return false; + } + if(!endsWith('"')) + { + qDebug() << "String fragment does not end with \", instead, there is" << at(size() - 1); + return false; + } + for(int i = 1; i < size() - 1; i++) + { + char c = operator[](i); + if(escape) + { + switch(c) + { + case 'r': + msg += '\r'; + break; + case 'n': + msg += '\n'; + break; + case 't': + msg += '\t'; + break; + case 'v': + msg += '\v'; + break; + case 'a': + msg += '\a'; + break; + case 'b': + msg += '\b'; + break; + case 'f': + msg += '\f'; + break; + case '"': + msg += '"'; + break; + case '\\': + msg.append('\\'); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { + int octal_start = i; + while ((c = operator[](i)) >= '0' && c <= '7') + { + i++; + if (i == length() - 1) + { + qDebug() << "Something went bad while parsing an octal escape string..."; + return false; + } + } + msg += mid(octal_start, i - octal_start).toUInt(0, 8); + break; + } + case 'x': + { + // chomp the 'x' + i++; + int hex_start = i; + while (isxdigit(operator[](i))) + { + i++; + if (i == length() - 1) + { + qDebug() << "Something went bad while parsing a hex escape string..."; + return false; + } + } + msg += mid(hex_start, i - hex_start).toUInt(0, 16); + break; + } + default: + { + qDebug() << "Invalid escape sequence character:" << c; + return false; + } + } + escape = false; + } + else if(c == '\\') + { + escape = true; + } + else + { + msg += c; + } + } + if(escape) + { + qDebug() << "Unterminated escape sequence..."; + return false; + } + appendHere += msg; + return true; + } +}; + +void POTranslatorPrivate::reload() +{ + QFile file(filename); + if(!file.open(QFile::OpenMode::enum_type::ReadOnly | QFile::OpenMode::enum_type::Text)) + { + qDebug() << "Failed to open PO file:" << filename; + return; + } + + QByteArray context; + QByteArray disambiguation; + QByteArray id; + QByteArray str; + bool fuzzy = false; + bool nextFuzzy = false; + + enum class Mode + { + First, + MessageContext, + MessageId, + MessageString + } mode = Mode::First; + + int lineNumber = 0; + QHash newMapping; + QHash newMapping_disambiguation; + auto endEntry = [&]() { + auto strStr = QString::fromUtf8(str); + // NOTE: PO header has empty id. We skip it. + if(!id.isEmpty()) + { + auto normalKey = context + "|" + id; + newMapping.insert(normalKey, {strStr, fuzzy}); + if(!disambiguation.isEmpty()) + { + auto disambiguationKey = context + "|" + id + "@" + disambiguation; + newMapping_disambiguation.insert(disambiguationKey, {strStr, fuzzy}); + } + } + context.clear(); + disambiguation.clear(); + id.clear(); + str.clear(); + fuzzy = nextFuzzy; + nextFuzzy = false; + }; + while (!file.atEnd()) + { + ParserArray line = file.readLine(); + if(line.endsWith('\n')) + { + line.resize(line.size() - 1); + } + if(line.endsWith('\r')) + { + line.resize(line.size() - 1); + } + + if(!line.size()) + { + // NIL + } + else if(line[0] == '#') + { + if(line.contains(", fuzzy")) + { + nextFuzzy = true; + } + } + else if(line.startsWith('"')) + { + QByteArray temp; + QByteArray *out = &temp; + + switch(mode) + { + case Mode::First: + qDebug() << "Unexpected escaped string during initial state... line:" << lineNumber; + return; + case Mode::MessageString: + out = &str; + break; + case Mode::MessageContext: + out = &context; + break; + case Mode::MessageId: + out = &id; + break; + } + if(!line.chompString(*out)) + { + qDebug() << "Badly formatted string on line:" << lineNumber; + return; + } + } + else if(line.chomp("msgctxt ", 8)) + { + switch(mode) + { + case Mode::First: + break; + case Mode::MessageString: + endEntry(); + break; + case Mode::MessageContext: + case Mode::MessageId: + qDebug() << "Unexpected msgctxt line:" << lineNumber; + return; + } + if(line.chompString(context)) + { + auto parts = context.split('|'); + context = parts[0]; + if(parts.size() > 1 && !parts[1].isEmpty()) + { + disambiguation = parts[1]; + } + mode = Mode::MessageContext; + } + } + else if (line.chomp("msgid ", 6)) + { + switch(mode) + { + case Mode::MessageContext: + case Mode::First: + break; + case Mode::MessageString: + endEntry(); + break; + case Mode::MessageId: + qDebug() << "Unexpected msgid line:" << lineNumber; + return; + } + if(line.chompString(id)) + { + mode = Mode::MessageId; + } + } + else if (line.chomp("msgstr ", 7)) + { + switch(mode) + { + case Mode::First: + case Mode::MessageString: + case Mode::MessageContext: + qDebug() << "Unexpected msgstr line:" << lineNumber; + return; + case Mode::MessageId: + break; + } + if(line.chompString(str)) + { + mode = Mode::MessageString; + } + } + else + { + qDebug() << "I did not understand line: " << lineNumber << ":" << QString::fromUtf8(line); + } + lineNumber++; + } + endEntry(); + mapping = std::move(newMapping); + mapping_disambiguatrion = std::move(newMapping_disambiguation); + loaded = true; +} + +POTranslator::POTranslator(const QString& filename, QObject* parent) : QTranslator(parent) +{ + d = new POTranslatorPrivate; + d->filename = filename; + d->reload(); +} + +QString POTranslator::translate(const char* context, const char* sourceText, const char* disambiguation, int n) const +{ + if(disambiguation) + { + auto disambiguationKey = QByteArray(context) + "|" + QByteArray(sourceText) + "@" + QByteArray(disambiguation); + auto iter = d->mapping_disambiguatrion.find(disambiguationKey); + if(iter != d->mapping_disambiguatrion.end()) + { + auto & entry = *iter; + if(entry.text.isEmpty()) + { + qDebug() << "Translation entry has no content:" << disambiguationKey; + } + if(entry.fuzzy) + { + qDebug() << "Translation entry is fuzzy:" << disambiguationKey << "->" << entry.text; + } + return entry.text; + } + } + auto key = QByteArray(context) + "|" + QByteArray(sourceText); + auto iter = d->mapping.find(key); + if(iter != d->mapping.end()) + { + auto & entry = *iter; + if(entry.text.isEmpty()) + { + qDebug() << "Translation entry has no content:" << key; + } + if(entry.fuzzy) + { + qDebug() << "Translation entry is fuzzy:" << key << "->" << entry.text; + } + return entry.text; + } + return QString(); +} + +bool POTranslator::isEmpty() const +{ + return !d->loaded; +} diff --git a/api/logic/translations/POTranslator.h b/api/logic/translations/POTranslator.h new file mode 100644 index 00000000..6d518560 --- /dev/null +++ b/api/logic/translations/POTranslator.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +struct POTranslatorPrivate; + +class POTranslator : public QTranslator +{ + Q_OBJECT +public: + explicit POTranslator(const QString& filename, QObject * parent = nullptr); + QString translate(const char * context, const char * sourceText, const char * disambiguation, int n) const override; + bool isEmpty() const override; +private: + POTranslatorPrivate * d; +}; diff --git a/api/logic/translations/TranslationsModel.cpp b/api/logic/translations/TranslationsModel.cpp index c32483f0..c9c30212 100644 --- a/api/logic/translations/TranslationsModel.cpp +++ b/api/logic/translations/TranslationsModel.cpp @@ -8,16 +8,66 @@ #include #include #include +#include #include #include +#include "Json.h" + +#include "POTranslator.h" const static QLatin1Literal defaultLangCode("en"); +enum class FileType +{ + NONE, + QM, + PO +}; + struct Language { + Language() + { + updated = true; + } + Language(const QString & _key) + { + key = _key; + locale = QLocale(key); + updated = (key == defaultLangCode); + } + + float percentTranslated() const + { + if (total == 0) + { + return 100.0f; + } + return float(translated) / float(total); + } + + void setTranslationStats(unsigned _translated, unsigned _untranslated, unsigned _fuzzy) + { + translated = _translated; + untranslated = _untranslated; + fuzzy = _fuzzy; + total = translated + untranslated + fuzzy; + } + QString key; QLocale locale; bool updated; + + QString file_name = QString(); + std::size_t file_size = 0; + QString file_sha1 = QString(); + + unsigned translated = 0; + unsigned untranslated = 0; + unsigned fuzzy = 0; + unsigned total = 0; + + FileType localFileType = FileType::NONE; }; struct TranslationsModel::Private @@ -25,7 +75,9 @@ struct TranslationsModel::Private QDir m_dir; // initial state is just english - QVector m_languages = {{defaultLangCode, QLocale(defaultLangCode), false}}; + QVector m_languages = {Language (defaultLangCode)}; + QMap m_languageLookup = {{defaultLangCode, 0}}; + QString m_selectedLanguage = defaultLangCode; std::unique_ptr m_qt_translator; std::unique_ptr m_app_translator; @@ -35,19 +87,168 @@ struct TranslationsModel::Private NetJobPtr m_dl_job; NetJobPtr m_index_job; QString m_nextDownload; + + std::unique_ptr m_po_translator; + QFileSystemWatcher *watcher; }; TranslationsModel::TranslationsModel(QString path, QObject* parent): QAbstractListModel(parent) { d.reset(new Private); d->m_dir.setPath(path); - loadLocalIndex(); + reloadLocalFiles(); + + d->watcher = new QFileSystemWatcher(this); + connect(d->watcher, &QFileSystemWatcher::directoryChanged, this, &TranslationsModel::translationDirChanged); + d->watcher->addPath(d->m_dir.canonicalPath()); } TranslationsModel::~TranslationsModel() { } +void TranslationsModel::translationDirChanged(const QString& path) +{ + qDebug() << "Dir changed:" << path; + reloadLocalFiles(); + selectLanguage(selectedLanguage()); +} + +void TranslationsModel::indexRecieved() +{ + qDebug() << "Got translations index!"; + d->m_index_job.reset(); + if(d->m_selectedLanguage != defaultLangCode) + { + downloadTranslation(d->m_selectedLanguage); + } +} + +namespace { +void readIndex(const QString & path, QVector& languages, QMap& languagesLookup) +{ + QByteArray data; + try + { + data = FS::read(path); + } + catch (const Exception &e) + { + qCritical() << "Translations Download Failed: index file not readable"; + return; + } + + int index = 1; + try + { + auto doc = Json::requireObject(Json::requireDocument(data)); + auto file_type = Json::requireString(doc, "file_type"); + if(file_type != "MMC-TRANSLATION-INDEX") + { + qCritical() << "Translations Download Failed: index file is of unknown file type" << file_type; + return; + } + auto version = Json::requireInteger(doc, "version"); + if(version > 2) + { + qCritical() << "Translations Download Failed: index file is of unknown format version" << file_type; + return; + } + auto langObjs = Json::requireObject(doc, "languages"); + for(auto iter = langObjs.begin(); iter != langObjs.end(); iter++) + { + Language lang(iter.key()); + + auto langObj = Json::requireObject(iter.value()); + lang.setTranslationStats( + Json::ensureInteger(langObj, "translated", 0), + Json::ensureInteger(langObj, "untranslated", 0), + Json::ensureInteger(langObj, "fuzzy", 0) + ); + lang.file_name = Json::requireString(langObj, "file"); + lang.file_sha1 = Json::requireString(langObj, "sha1"); + lang.file_size = Json::requireInteger(langObj, "size"); + + languages.append(std::move(lang)); + languagesLookup[iter.key()] = index; + index++; + } + } + catch (Json::JsonException & e) + { + qCritical() << "Translations Download Failed: index file could not be parsed as json"; + } +} +} + +void TranslationsModel::reloadLocalFiles() +{ + QVector languages = {Language (defaultLangCode)}; + QMap languageLookup = {{defaultLangCode, 0}}; + + readIndex(d->m_dir.absoluteFilePath("index_v2.json"), languages, languageLookup); + auto fileTypeToString = [](FileType ft) -> QString + { + switch(ft) + { + case FileType::NONE: + return QString(); + case FileType::QM: + return "QM"; + case FileType::PO: + return "PO"; + } + return QString(); + }; + auto entries = d->m_dir.entryInfoList({"mmc_*.qm", "*.po"}, QDir::Files | QDir::NoDotAndDotDot); + for(auto & entry: entries) + { + auto completeSuffix = entry.completeSuffix(); + QString langCode; + FileType fileType = FileType::NONE; + if(completeSuffix == "qm") + { + langCode = entry.baseName().remove(0,4); + fileType = FileType::QM; + } + else if(completeSuffix == "po") + { + langCode = entry.baseName(); + fileType = FileType::PO; + } + else + { + continue; + } + + auto langIter = languageLookup.find(langCode); + if(langIter != languageLookup.end()) + { + auto & language = languages[*langIter]; + if(int(fileType) > int(language.localFileType)) + { + qDebug() << "Found" << fileTypeToString(fileType) << "local file for language" << langCode; + language.localFileType = fileType; + } + } + else + { + if(fileType == FileType::PO) + { + Language localFound(langCode); + localFound.localFileType = FileType::PO; + languages.append(localFound); + qDebug() << "Found standalone translation PO file: " << langCode; + } + } + + } + + beginResetModel(); + d->m_languages.swap(languages); + endResetModel(); +} + QVariant TranslationsModel::data(const QModelIndex& index, int role) const { if (!index.isValid()) @@ -153,18 +354,48 @@ bool TranslationsModel::selectLanguage(QString key) d->m_qt_translator.reset(); } - d->m_app_translator.reset(new QTranslator()); - if (d->m_app_translator->load("mmc_" + langCode, d->m_dir.path())) + if(langPtr->localFileType == FileType::PO) { qDebug() << "Loading Application Language File for" << langCode.toLocal8Bit().constData() << "..."; - if (!QCoreApplication::installTranslator(d->m_app_translator.get())) + auto poTranslator = new POTranslator(FS::PathCombine(d->m_dir.path(), langCode + ".po")); + if(!poTranslator->isEmpty()) + { + if (!QCoreApplication::installTranslator(poTranslator)) + { + delete poTranslator; + qCritical() << "Installing Application Language File failed."; + } + else + { + d->m_app_translator.reset(poTranslator); + successful = true; + } + } + else { qCritical() << "Loading Application Language File failed."; d->m_app_translator.reset(); } + } + else if(langPtr->localFileType == FileType::QM) + { + d->m_app_translator.reset(new QTranslator()); + if (d->m_app_translator->load("mmc_" + langCode, d->m_dir.path())) + { + qDebug() << "Loading Application Language File for" << langCode.toLocal8Bit().constData() << "..."; + if (!QCoreApplication::installTranslator(d->m_app_translator.get())) + { + qCritical() << "Installing Application Language File failed."; + d->m_app_translator.reset(); + } + else + { + successful = true; + } + } else { - successful = true; + d->m_app_translator.reset(); } } else @@ -199,55 +430,15 @@ void TranslationsModel::downloadIndex() } qDebug() << "Downloading Translations Index..."; d->m_index_job.reset(new NetJob("Translations Index")); - MetaEntryPtr entry = ENV.metacache()->resolveEntry("translations", "index"); - d->m_index_task = Net::Download::makeCached(QUrl("https://files.multimc.org/translations/index"), entry); + MetaEntryPtr entry = ENV.metacache()->resolveEntry("translations", "index_v2.json"); + entry->setStale(true); + d->m_index_task = Net::Download::makeCached(QUrl("https://files.multimc.org/translations/index_v2.json"), entry); d->m_index_job->addNetAction(d->m_index_task); connect(d->m_index_job.get(), &NetJob::failed, this, &TranslationsModel::indexFailed); connect(d->m_index_job.get(), &NetJob::succeeded, this, &TranslationsModel::indexRecieved); d->m_index_job->start(); } -void TranslationsModel::indexRecieved() -{ - qDebug() << "Got translations index!"; - d->m_index_job.reset(); - loadLocalIndex(); - if(d->m_selectedLanguage != defaultLangCode) - { - downloadTranslation(d->m_selectedLanguage); - } -} - -void TranslationsModel::loadLocalIndex() -{ - QByteArray data; - try - { - data = FS::read(d->m_dir.absoluteFilePath("index")); - } - catch (const Exception &e) - { - qCritical() << "Translations Download Failed: index file not readable"; - return; - } - QVector languages; - QList lines = data.split('\n'); - // add the default english. - languages.append({defaultLangCode, QLocale(defaultLangCode), true}); - for (const auto line : lines) - { - if(!line.isEmpty()) - { - auto str = QString::fromLatin1(line); - str.remove(".qm"); - languages.append({str, QLocale(str), false}); - } - } - beginResetModel(); - d->m_languages.swap(languages); - endResetModel(); -} - void TranslationsModel::updateLanguage(QString key) { if(key == defaultLangCode) @@ -274,13 +465,28 @@ void TranslationsModel::downloadTranslation(QString key) d->m_nextDownload = key; return; } + auto lang = findLanguage(key); + if(!lang) + { + qWarning() << "Will not download an unknown translation" << key; + return; + } + d->m_downloadingTranslation = key; MetaEntryPtr entry = ENV.metacache()->resolveEntry("translations", "mmc_" + key + ".qm"); entry->setStale(true); + + auto dl = Net::Download::makeCached(QUrl(URLConstants::TRANSLATIONS_BASE_URL + lang->file_name), entry); + auto rawHash = QByteArray::fromHex(lang->file_sha1.toLatin1()); + dl->addValidator(new Net::ChecksumValidator(QCryptographicHash::Sha1, rawHash)); + dl->m_total_progress = lang->file_size; + d->m_dl_job.reset(new NetJob("Translation for " + key)); - d->m_dl_job->addNetAction(Net::Download::makeCached(QUrl(URLConstants::TRANSLATIONS_BASE_URL + key + ".qm"), entry)); + d->m_dl_job->addNetAction(dl); + connect(d->m_dl_job.get(), &NetJob::succeeded, this, &TranslationsModel::dlGood); connect(d->m_dl_job.get(), &NetJob::failed, this, &TranslationsModel::dlFailed); + d->m_dl_job->start(); } diff --git a/api/logic/translations/TranslationsModel.h b/api/logic/translations/TranslationsModel.h index 8a9298d4..5bf769c3 100644 --- a/api/logic/translations/TranslationsModel.h +++ b/api/logic/translations/TranslationsModel.h @@ -40,7 +40,7 @@ public: private: Language *findLanguage(const QString & key); - void loadLocalIndex(); + void reloadLocalFiles(); void downloadTranslation(QString key); void downloadNext(); @@ -54,6 +54,8 @@ private slots: void indexFailed(QString reason); void dlFailed(QString reason); void dlGood(); + void translationDirChanged(const QString &path); + private: /* data */ struct Private;