feat: add mod update check tasks

Those tasks take a list of mods and check on the mod providers for
updates. They assume that the mods have metadata already.

Signed-off-by: flow <flowlnlnln@gmail.com>
This commit is contained in:
flow 2022-06-03 21:26:26 -03:00
parent c3f6c3dd82
commit b8b71c7dd2
No known key found for this signature in database
GPG Key ID: 8D0F221F0A59F469
8 changed files with 499 additions and 1 deletions

View File

@ -485,6 +485,8 @@ set(API_SOURCES
modplatform/EnsureMetadataTask.h modplatform/EnsureMetadataTask.h
modplatform/EnsureMetadataTask.cpp modplatform/EnsureMetadataTask.cpp
modplatform/CheckUpdateTask.h
modplatform/flame/FlameAPI.h modplatform/flame/FlameAPI.h
modplatform/flame/FlameAPI.cpp modplatform/flame/FlameAPI.cpp
modplatform/modrinth/ModrinthAPI.h modplatform/modrinth/ModrinthAPI.h
@ -514,6 +516,8 @@ set(FLAME_SOURCES
modplatform/flame/PackManifest.cpp modplatform/flame/PackManifest.cpp
modplatform/flame/FileResolvingTask.h modplatform/flame/FileResolvingTask.h
modplatform/flame/FileResolvingTask.cpp modplatform/flame/FileResolvingTask.cpp
modplatform/flame/FlameCheckUpdate.cpp
modplatform/flame/FlameCheckUpdate.h
) )
set(MODRINTH_SOURCES set(MODRINTH_SOURCES
@ -521,6 +525,8 @@ set(MODRINTH_SOURCES
modplatform/modrinth/ModrinthPackIndex.h modplatform/modrinth/ModrinthPackIndex.h
modplatform/modrinth/ModrinthPackManifest.cpp modplatform/modrinth/ModrinthPackManifest.cpp
modplatform/modrinth/ModrinthPackManifest.h modplatform/modrinth/ModrinthPackManifest.h
modplatform/modrinth/ModrinthCheckUpdate.cpp
modplatform/modrinth/ModrinthCheckUpdate.h
) )
set(MODPACKSCH_SOURCES set(MODPACKSCH_SOURCES

View File

@ -0,0 +1,51 @@
#pragma once
#include "minecraft/mod/Mod.h"
#include "modplatform/ModAPI.h"
#include "modplatform/ModIndex.h"
#include "tasks/Task.h"
class ModDownloadTask;
class ModFolderModel;
class CheckUpdateTask : public Task {
Q_OBJECT
public:
CheckUpdateTask(std::list<Mod>& mods, std::list<Version>& mcVersions, ModAPI::ModLoaderTypes loaders, std::shared_ptr<ModFolderModel> mods_folder)
: m_mods(mods), m_game_versions(mcVersions), m_loaders(loaders), m_mods_folder(mods_folder) {};
struct UpdatableMod {
QString name;
QString old_hash;
QString old_version;
QString new_version;
QString changelog;
ModPlatform::Provider provider;
ModDownloadTask* download;
public:
UpdatableMod(QString name, QString old_h, QString old_v, QString new_v, QString changelog, ModPlatform::Provider p, ModDownloadTask* t)
: name(name), old_hash(old_h), old_version(old_v), new_version(new_v), changelog(changelog), provider(p), download(t)
{}
};
auto getUpdatable() -> std::vector<UpdatableMod>&& { return std::move(m_updatable); }
public slots:
bool abort() override = 0;
protected slots:
void executeTask() override = 0;
signals:
void checkFailed(Mod failed, QString reason, QUrl recover_url = {});
protected:
std::list<Mod>& m_mods;
std::list<Version>& m_game_versions;
ModAPI::ModLoaderTypes m_loaders;
std::shared_ptr<ModFolderModel> m_mods_folder;
std::vector<UpdatableMod> m_updatable;
};

View File

@ -54,6 +54,7 @@ struct IndexedVersion {
QVariant addonId; QVariant addonId;
QVariant fileId; QVariant fileId;
QString version; QString version;
QString version_number = {};
QVector<QString> mcVersion; QVector<QString> mcVersion;
QString downloadUrl; QString downloadUrl;
QString date; QString date;

View File

@ -0,0 +1,228 @@
#include "FlameCheckUpdate.h"
#include "FlameAPI.h"
#include "FlameModIndex.h"
#include <MurmurHash2.h>
#include "FileSystem.h"
#include "Json.h"
#include "ModDownloadTask.h"
static FlameAPI api;
static ModPlatform::ProviderCapabilities ProviderCaps;
bool FlameCheckUpdate::abort()
{
m_was_aborted = true;
if (m_net_job)
return m_net_job->abort();
return true;
}
ModPlatform::IndexedPack getProjectInfo(ModPlatform::IndexedVersion& ver_info)
{
ModPlatform::IndexedPack pack;
QEventLoop loop;
auto get_project_job = new NetJob("Flame::GetProjectJob", APPLICATION->network());
auto response = new QByteArray();
auto url = QString("https://api.curseforge.com/v1/mods/%1").arg(ver_info.addonId.toString());
auto dl = Net::Download::makeByteArray(url, response);
get_project_job->addNetAction(dl);
QObject::connect(get_project_job, &NetJob::succeeded, [response, &pack]() {
QJsonParseError parse_error{};
QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from FlameCheckUpdate at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
return;
}
try {
auto doc_obj = Json::requireObject(doc);
auto data_obj = Json::requireObject(doc_obj, "data");
FlameMod::loadIndexedPack(pack, data_obj);
} catch (Json::JsonException& e) {
qWarning() << e.cause();
qDebug() << doc;
}
});
QObject::connect(get_project_job, &NetJob::finished, [&loop, get_project_job] {
get_project_job->deleteLater();
loop.quit();
});
get_project_job->start();
loop.exec();
return pack;
}
/* Check for update:
* - Get latest version available
* - Compare hash of the latest version with the current hash
* - If equal, no updates, else, there's updates, so add to the list
* */
void FlameCheckUpdate::executeTask()
{
setStatus(tr("Preparing mods for CurseForge..."));
setProgress(0, 5);
QHash<QString, Mod> mappings;
// Create all hashes
QStringList hashes;
std::list<uint> murmur_hashes;
auto best_hash_type = ProviderCaps.hashType(ModPlatform::Provider::FLAME).first();
for (auto mod : m_mods) {
auto hash = mod.metadata()->hash;
QByteArray jar_data;
try {
jar_data = FS::read(mod.fileinfo().absoluteFilePath());
} catch (FS::FileSystemException& e) {
qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name());
qCritical() << QString("Reason: ") << e.cause();
failed(e.what());
return;
}
QByteArray jar_data_treated;
for (char c : jar_data) {
// CF-specific
if (!(c == 9 || c == 10 || c == 13 || c == 32))
jar_data_treated.push_back(c);
}
auto murmur_hash = MurmurHash2(jar_data_treated, jar_data_treated.length());
murmur_hashes.emplace_back(murmur_hash);
// Sadly the API can only handle one hash type per call, se we
// need to generate a new hash if the current one is innadequate
// (though it will rarely happen, if at all)
if (mod.metadata()->hash_format != best_hash_type)
hash = QString(ProviderCaps.hash(ModPlatform::Provider::FLAME, jar_data, best_hash_type).toHex());
hashes.append(hash);
mappings.insert(hash, mod);
}
auto* response = new QByteArray();
auto job = api.matchFingerprints(murmur_hashes, response);
QEventLoop lock;
connect(job.get(), &Task::succeeded, this, [this, response, &mappings] {
QJsonParseError parse_error{};
QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from FlameCheckUpdate at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
failed(parse_error.errorString());
return;
}
setStatus(tr("Parsing the first API response from CurseForge..."));
setProgress(2, 5);
try {
auto doc_obj = Json::requireObject(doc);
auto data_obj = Json::ensureObject(doc_obj, "data");
auto match_arr = Json::ensureArray(data_obj, "exactMatches");
for (auto match : match_arr) {
auto match_obj = Json::ensureObject(match);
ModPlatform::IndexedVersion current_ver;
try {
auto file_obj = Json::requireObject(match_obj, "file");
current_ver = FlameMod::loadIndexedPackVersion(file_obj);
} catch (Json::JsonException& e) {
qCritical() << "Error while parsing Flame indexed version";
qCritical() << e.what();
failed(tr("An error occured while parsing a CurseForge indexed version!"));
return;
}
auto mod_iter = mappings.find(current_ver.hash);
if (mod_iter == mappings.end()) {
qCritical() << "Failed to remap mod from Flame!";
continue;
}
auto mod = mod_iter.value();
setStatus(tr("Waiting for the API response from CurseForge for '%1'...").arg(mod.name()));
setProgress(3, 5);
auto latest_ver = api.getLatestVersion({ current_ver.addonId.toString(), m_game_versions, m_loaders });
// Check if we were aborted while getting the latest version
if (m_was_aborted) {
aborted();
return;
}
setStatus(tr("Parsing the API response from CurseForge for '%1'...").arg(mod.name()));
setProgress(4, 5);
if (!latest_ver.addonId.isValid()) {
emit checkFailed(
mod,
tr("No valid version found for this mod. It's probably unavailable for the current game version / mod loader."));
continue;
}
if (latest_ver.downloadUrl.isEmpty() && latest_ver.fileId != current_ver.fileId) {
auto pack = getProjectInfo(latest_ver);
auto recover_url = QString("%1/download/%2").arg(pack.websiteUrl, latest_ver.fileId.toString());
emit checkFailed(mod, tr("Mod has a new update available, but is opted-out on CurseForge"), recover_url);
continue;
}
if (!latest_ver.hash.isEmpty() && current_ver.hash != latest_ver.hash) {
// Fake pack with the necessary info to pass to the download task :)
ModPlatform::IndexedPack pack;
pack.name = mod.name();
pack.addonId = mod.metadata()->project_id;
pack.websiteUrl = mod.homeurl();
for (auto& author : mod.authors())
pack.authors.append({ author });
pack.description = mod.description();
pack.provider = ModPlatform::Provider::FLAME;
auto download_task = new ModDownloadTask(pack, latest_ver, m_mods_folder);
m_updatable.emplace_back(mod.name(), current_ver.hash, current_ver.version, latest_ver.version,
api.getModFileChangelog(latest_ver.addonId.toInt(), latest_ver.fileId.toInt()),
ModPlatform::Provider::FLAME, download_task);
}
}
} catch (Json::JsonException& e) {
failed(e.cause() + " : " + e.what());
}
});
connect(job.get(), &Task::finished, &lock, &QEventLoop::quit);
setStatus(tr("Waiting for the first API response from CurseForge..."));
setProgress(1, 5);
m_net_job = job.get();
job->start();
lock.exec();
emitSucceeded();
}

View File

@ -0,0 +1,25 @@
#pragma once
#include "Application.h"
#include "modplatform/CheckUpdateTask.h"
#include "net/NetJob.h"
class FlameCheckUpdate : public CheckUpdateTask {
Q_OBJECT
public:
FlameCheckUpdate(std::list<Mod>& mods, std::list<Version>& mcVersions, ModAPI::ModLoaderTypes loaders, std::shared_ptr<ModFolderModel> mods_folder)
: CheckUpdateTask(mods, mcVersions, loaders, mods_folder)
{}
public slots:
bool abort() override;
protected slots:
void executeTask() override;
private:
NetJob* m_net_job = nullptr;
bool m_was_aborted = false;
};

View File

@ -0,0 +1,163 @@
#include "ModrinthCheckUpdate.h"
#include "ModrinthAPI.h"
#include "ModrinthPackIndex.h"
#include "FileSystem.h"
#include "Json.h"
#include "ModDownloadTask.h"
static ModrinthAPI api;
static ModPlatform::ProviderCapabilities ProviderCaps;
bool ModrinthCheckUpdate::abort()
{
if (m_net_job)
return m_net_job->abort();
return true;
}
/* Check for update:
* - Get latest version available
* - Compare hash of the latest version with the current hash
* - If equal, no updates, else, there's updates, so add to the list
* */
void ModrinthCheckUpdate::executeTask()
{
setStatus(tr("Preparing mods for Modrinth..."));
setProgress(0, 3);
QHash<QString, Mod> mappings;
// Create all hashes
QStringList hashes;
auto best_hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first();
for (auto mod : m_mods) {
auto hash = mod.metadata()->hash;
// Sadly the API can only handle one hash type per call, se we
// need to generate a new hash if the current one is innadequate
// (though it will rarely happen, if at all)
if (mod.metadata()->hash_format != best_hash_type) {
QByteArray jar_data;
try {
jar_data = FS::read(mod.fileinfo().absoluteFilePath());
} catch (FS::FileSystemException& e) {
qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name());
qCritical() << QString("Reason: ") << e.cause();
failed(e.what());
return;
}
hash = QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, best_hash_type).toHex());
}
hashes.append(hash);
mappings.insert(hash, mod);
}
auto* response = new QByteArray();
auto job = api.latestVersions(hashes, best_hash_type, m_game_versions, m_loaders, response);
QEventLoop lock;
connect(job.get(), &Task::succeeded, this, [this, response, &mappings, best_hash_type, job] {
QJsonParseError parse_error{};
QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from ModrinthCheckUpdate at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
failed(parse_error.errorString());
return;
}
setStatus(tr("Parsing the API response from Modrinth..."));
setProgress(2, 3);
try {
for (auto hash : mappings.keys()) {
auto project_obj = doc[hash].toObject();
if (project_obj.isEmpty()) {
qDebug() << "Mod " << mappings.find(hash).value().name() << " got an empty response.";
qDebug() << "Hash: " << hash;
emit checkFailed(mappings.find(hash).value(), tr("Couldn't find mod in Modrinth"));
continue;
}
// Sometimes a version may have multiple files, one with "forge" and one with "fabric",
// so we may want to filter it
QString loader_filter;
static auto flags = { ModAPI::ModLoaderType::Forge, ModAPI::ModLoaderType::Fabric, ModAPI::ModLoaderType::Quilt };
for (auto flag : flags) {
if (m_loaders.testFlag(flag)) {
loader_filter = api.getModLoaderString(flag);
break;
}
}
// Currently, we rely on a couple heuristics to determine whether an update is actually available or not:
// - The file needs to be preferred: It is either the primary file, or the one found via (explicit) usage of the loader_filter
// - The version reported by the JAR is different from the version reported by the indexed version (it's usually the case)
// Such is the pain of having arbitrary files for a given version .-.
auto project_ver = Modrinth::loadIndexedPackVersion(project_obj, best_hash_type, loader_filter);
if (project_ver.downloadUrl.isEmpty()) {
qCritical() << "Modrinth mod without download url!";
qCritical() << project_ver.fileName;
emit checkFailed(mappings.find(hash).value(), tr("Mod has an empty download URL"));
continue;
}
auto mod_iter = mappings.find(hash);
if (mod_iter == mappings.end()) {
qCritical() << "Failed to remap mod from Modrinth!";
continue;
}
auto mod = *mod_iter;
auto key = project_ver.hash;
if ((key != hash && project_ver.is_preferred) || (mod.status() == ModStatus::NotInstalled)) {
if (mod.version() == project_ver.version_number)
continue;
// Fake pack with the necessary info to pass to the download task :)
ModPlatform::IndexedPack pack;
pack.name = mod.name();
pack.addonId = mod.metadata()->project_id;
pack.websiteUrl = mod.homeurl();
for (auto& author : mod.authors())
pack.authors.append({ author });
pack.description = mod.description();
pack.provider = ModPlatform::Provider::MODRINTH;
auto download_task = new ModDownloadTask(pack, project_ver, m_mods_folder);
m_updatable.emplace_back(mod.name(), hash, mod.version(), project_ver.version_number, project_ver.changelog,
ModPlatform::Provider::MODRINTH, download_task);
}
}
} catch (Json::JsonException& e) {
failed(e.cause() + " : " + e.what());
}
});
connect(job.get(), &Task::finished, &lock, &QEventLoop::quit);
setStatus(tr("Waiting for the API response from Modrinth..."));
setProgress(1, 3);
m_net_job = job.get();
job->start();
lock.exec();
emitSucceeded();
}

View File

@ -0,0 +1,23 @@
#pragma once
#include "Application.h"
#include "modplatform/CheckUpdateTask.h"
#include "net/NetJob.h"
class ModrinthCheckUpdate : public CheckUpdateTask {
Q_OBJECT
public:
ModrinthCheckUpdate(std::list<Mod>& mods, std::list<Version>& mcVersions, ModAPI::ModLoaderTypes loaders, std::shared_ptr<ModFolderModel> mods_folder)
: CheckUpdateTask(mods, mcVersions, loaders, mods_folder)
{}
public slots:
bool abort() override;
protected slots:
void executeTask() override;
private:
NetJob* m_net_job = nullptr;
};

View File

@ -130,6 +130,7 @@ auto Modrinth::loadIndexedPackVersion(QJsonObject &obj, QString preferred_hash_t
file.loaders.append(loader.toString()); file.loaders.append(loader.toString());
} }
file.version = Json::requireString(obj, "name"); file.version = Json::requireString(obj, "name");
file.version_number = Json::requireString(obj, "version_number");
file.changelog = Json::requireString(obj, "changelog"); file.changelog = Json::requireString(obj, "changelog");
auto files = Json::requireArray(obj, "files"); auto files = Json::requireArray(obj, "files");
@ -159,7 +160,7 @@ auto Modrinth::loadIndexedPackVersion(QJsonObject &obj, QString preferred_hash_t
if (parent.contains("url")) { if (parent.contains("url")) {
file.downloadUrl = Json::requireString(parent, "url"); file.downloadUrl = Json::requireString(parent, "url");
file.fileName = Json::requireString(parent, "filename"); file.fileName = Json::requireString(parent, "filename");
file.is_preferred = Json::requireBoolean(parent, "primary"); file.is_preferred = Json::requireBoolean(parent, "primary") || (files.count() == 1);
auto hash_list = Json::requireObject(parent, "hashes"); auto hash_list = Json::requireObject(parent, "hashes");
if (hash_list.contains(preferred_hash_type)) { if (hash_list.contains(preferred_hash_type)) {