From 9d03a9c1e3b9c24a4146adedb2971591d23b037a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 22 Sep 2013 14:00:37 +0200 Subject: [PATCH] Cache forge version list (it's huge) --- logic/ForgeInstaller.cpp | 12 +++- logic/NostalgiaInstance.h | 1 + logic/OneSixInstance.cpp | 1 + logic/lists/ForgeVersionList.cpp | 119 ++++++++++++++++--------------- logic/net/CacheDownload.cpp | 63 ++++++++-------- logic/net/HttpMetaCache.h | 19 ++--- 6 files changed, 118 insertions(+), 97 deletions(-) diff --git a/logic/ForgeInstaller.cpp b/logic/ForgeInstaller.cpp index 00ad8d19..bcba00e9 100644 --- a/logic/ForgeInstaller.cpp +++ b/logic/ForgeInstaller.cpp @@ -1,10 +1,12 @@ #include "ForgeInstaller.h" #include "OneSixVersion.h" #include "OneSixLibrary.h" +#include "net/HttpMetaCache.h" #include #include #include #include +#include "MultiMC.h" ForgeInstaller::ForgeInstaller(QString filename, QString universal_url) { @@ -53,6 +55,8 @@ ForgeInstaller::ForgeInstaller(QString filename, QString universal_url) // where do we put the library? decode the mojang path OneSixLibrary lib(libraryName); lib.finalize(); + + auto cacheentry = MMC->metacache()->resolveEntry("libraries", lib.storagePath()); finalPath = "libraries/" + lib.storagePath(); if (!ensureFilePathExists(finalPath)) return; @@ -71,6 +75,12 @@ ForgeInstaller::ForgeInstaller(QString filename, QString universal_url) return; if (!extraction.commit()) return; + QCryptographicHash md5sum(QCryptographicHash::Md5); + md5sum.addData(data); + + cacheentry->stale = false; + cacheentry->md5sum = md5sum.result().toHex().constData(); + MMC->metacache()->updateEntry(cacheentry); } file.close(); @@ -91,7 +101,7 @@ bool ForgeInstaller::apply(QSharedPointer to) { QString libName = lib->name(); // if this is the actual forge lib, set an absolute url for the download - if(libName.contains("minecraftforge")) + if (libName.contains("minecraftforge")) { lib->setAbsoluteUrl(m_universal_url); } diff --git a/logic/NostalgiaInstance.h b/logic/NostalgiaInstance.h index f2df1828..1436e48d 100644 --- a/logic/NostalgiaInstance.h +++ b/logic/NostalgiaInstance.h @@ -9,3 +9,4 @@ public: explicit NostalgiaInstance(const QString &rootDir, SettingsObject * settings, QObject *parent = 0); virtual QString getStatusbarDescription(); }; + diff --git a/logic/OneSixInstance.cpp b/logic/OneSixInstance.cpp index 9262b155..e22a8890 100644 --- a/logic/OneSixInstance.cpp +++ b/logic/OneSixInstance.cpp @@ -320,3 +320,4 @@ QString OneSixInstance::instanceConfigFolder() const { return PathCombine(minecraftRoot(), "config"); } + diff --git a/logic/lists/ForgeVersionList.cpp b/logic/lists/ForgeVersionList.cpp index 9205e70f..721f2c0a 100644 --- a/logic/lists/ForgeVersionList.cpp +++ b/logic/lists/ForgeVersionList.cpp @@ -3,7 +3,7 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software @@ -25,10 +25,8 @@ #define JSON_URL "http://files.minecraftforge.net/minecraftforge/json" - -ForgeVersionList::ForgeVersionList(QObject* parent): BaseVersionList(parent) +ForgeVersionList::ForgeVersionList(QObject *parent) : BaseVersionList(parent) { - } Task *ForgeVersionList::getLoadTask() @@ -51,19 +49,19 @@ int ForgeVersionList::count() const return m_vlist.count(); } -int ForgeVersionList::columnCount(const QModelIndex& parent) const +int ForgeVersionList::columnCount(const QModelIndex &parent) const { - return 3; + return 3; } QVariant ForgeVersionList::data(const QModelIndex &index, int role) const { if (!index.isValid()) return QVariant(); - + if (index.row() > count()) return QVariant(); - + auto version = m_vlist[index.row()].dynamicCast(); switch (role) { @@ -72,22 +70,22 @@ QVariant ForgeVersionList::data(const QModelIndex &index, int role) const { case 0: return version->name(); - + case 1: return version->mcver; - + case 2: return version->typeString(); default: return QVariant(); } - + case Qt::ToolTipRole: return version->descriptor(); - + case VersionPointerRole: return qVariantFromValue(m_vlist[index.row()]); - + default: return QVariant(); } @@ -102,33 +100,33 @@ QVariant ForgeVersionList::headerData(int section, Qt::Orientation orientation, { case 0: return "Version"; - + case 1: return "Minecraft"; - + case 2: return "Type"; - + default: return QVariant(); } - + case Qt::ToolTipRole: switch (section) { case 0: return "The name of the version."; - + case 1: return "Minecraft version"; - + case 2: return "The version's type."; - + default: return QVariant(); } - + default: return QVariant(); } @@ -139,7 +137,7 @@ BaseVersionPtr ForgeVersionList::getLatestStable() const return BaseVersionPtr(); } -void ForgeVersionList::updateListData(QList versions) +void ForgeVersionList::updateListData(QList versions) { beginResetModel(); m_vlist = versions; @@ -154,104 +152,112 @@ void ForgeVersionList::sort() // NO-OP for now } - -ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList* vlist): Task() +ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList *vlist) : Task() { m_list = vlist; } - void ForgeListLoadTask::executeTask() { auto job = new DownloadJob("Version index"); - job->add(QUrl(JSON_URL)); + // we do not care if the version is stale or not. + auto forgeListEntry = MMC->metacache()->resolveEntry("minecraftforge", "list.json"); + job->add(QUrl(JSON_URL), forgeListEntry); listJob.reset(job); connect(listJob.data(), SIGNAL(succeeded()), SLOT(list_downloaded())); connect(listJob.data(), SIGNAL(failed()), SLOT(versionFileFailed())); - connect(listJob.data(), SIGNAL(progress(qint64,qint64)), SIGNAL(progress(qint64,qint64))); + connect(listJob.data(), SIGNAL(progress(qint64, qint64)), SIGNAL(progress(qint64, qint64))); listJob->start(); } void ForgeListLoadTask::list_downloaded() { - auto DlJob = listJob->first(); - auto data = DlJob.dynamicCast()->m_data; - - + QByteArray data; + { + auto DlJob = listJob->first(); + auto filename = DlJob.dynamicCast()->m_target_path; + QFile listFile(filename); + if(!listFile.open(QIODevice::ReadOnly)) + return; + data = listFile.readAll(); + DlJob.reset(); + } + QJsonParseError jsonError; QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError); - DlJob.reset(); - + + if (jsonError.error != QJsonParseError::NoError) { emitFailed("Error parsing version list JSON:" + jsonError.errorString()); return; } - if(!jsonDoc.isObject()) + if (!jsonDoc.isObject()) { emitFailed("Error parsing version list JSON: jsonDoc is not an object"); return; } - + QJsonObject root = jsonDoc.object(); - + // Now, get the array of versions. - if(!root.value("builds").isArray()) + if (!root.value("builds").isArray()) { - emitFailed("Error parsing version list JSON: version list object is missing 'builds' array"); + emitFailed( + "Error parsing version list JSON: version list object is missing 'builds' array"); return; } QJsonArray builds = root.value("builds").toArray(); - - QList tempList; + + QList tempList; for (int i = 0; i < builds.count(); i++) { // Load the version info. - if(!builds[i].isObject()) + if (!builds[i].isObject()) { - //FIXME: log this somewhere + // FIXME: log this somewhere continue; } QJsonObject obj = builds[i].toObject(); int build_nr = obj.value("build").toDouble(0); - if(!build_nr) + if (!build_nr) continue; QJsonArray files = obj.value("files").toArray(); QString url, jobbuildver, mcver, buildtype, filename; QString changelog_url, installer_url; QString installer_filename; bool valid = false; - for(int j = 0; j < files.count(); j++) + for (int j = 0; j < files.count(); j++) { - if(!files[j].isObject()) + if (!files[j].isObject()) continue; QJsonObject file = files[j].toObject(); buildtype = file.value("buildtype").toString(); - if((buildtype == "client" || buildtype == "universal") && !valid) + if ((buildtype == "client" || buildtype == "universal") && !valid) { mcver = file.value("mcver").toString(); url = file.value("url").toString(); jobbuildver = file.value("jobbuildver").toString(); int lastSlash = url.lastIndexOf('/'); - filename = url.mid(lastSlash+1); + filename = url.mid(lastSlash + 1); valid = true; } - else if(buildtype == "changelog") + else if (buildtype == "changelog") { QString ext = file.value("ext").toString(); - if(ext.isEmpty()) + if (ext.isEmpty()) continue; changelog_url = file.value("url").toString(); } - else if(buildtype == "installer") + else if (buildtype == "installer") { installer_url = file.value("url").toString(); int lastSlash = installer_url.lastIndexOf('/'); - installer_filename = installer_url.mid(lastSlash+1); + installer_filename = installer_url.mid(lastSlash + 1); } } - if(valid) + if (valid) { // Now, we construct the version object and add it to the list. QSharedPointer fVersion(new ForgeVersion()); @@ -260,7 +266,7 @@ void ForgeListLoadTask::list_downloaded() fVersion->installer_url = installer_url; fVersion->jobbuildver = jobbuildver; fVersion->mcver = mcver; - if(installer_filename.isEmpty()) + if (installer_filename.isEmpty()) fVersion->filename = filename; else fVersion->filename = installer_filename; @@ -269,12 +275,7 @@ void ForgeListLoadTask::list_downloaded() } } m_list->updateListData(tempList); - + emitSucceeded(); return; } - - - - - diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index c0074574..dc2e0448 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -7,8 +7,8 @@ #include #include -CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry ) - :Download(), md5sum(QCryptographicHash::Md5) +CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) + : Download(), md5sum(QCryptographicHash::Md5) { m_url = url; m_entry = entry; @@ -19,38 +19,40 @@ CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry ) void CacheDownload::start() { - if(!m_entry->stale) + if (!m_entry->stale) { emit succeeded(index_within_job); return; } m_output_file.setFileName(m_target_path); // if there already is a file and md5 checking is in effect and it can be opened - if(!ensureFilePathExists(m_target_path)) + if (!ensureFilePathExists(m_target_path)) { emit failed(index_within_job); return; } qDebug() << "Downloading " << m_url.toString(); - QNetworkRequest request ( m_url ); - request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); - + QNetworkRequest request(m_url); + request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); + auto worker = MMC->qnam(); - QNetworkReply * rep = worker->get ( request ); - - m_reply = QSharedPointer ( rep, &QObject::deleteLater ); - connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); - connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); - connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); - connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); + QNetworkReply *rep = worker->get(request); + + m_reply = QSharedPointer(rep, &QObject::deleteLater); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); } -void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +void CacheDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) { - emit progress (index_within_job, bytesReceived, bytesTotal ); + emit progress(index_within_job, bytesReceived, bytesTotal); } -void CacheDownload::downloadError ( QNetworkReply::NetworkError error ) +void CacheDownload::downloadError(QNetworkReply::NetworkError error) { // error happened during download. // TODO: log the reason why @@ -59,12 +61,12 @@ void CacheDownload::downloadError ( QNetworkReply::NetworkError error ) void CacheDownload::downloadFinished() { // if the download succeeded - if ( m_status != Job_Failed ) + if (m_status != Job_Failed) { - + // nothing went wrong... m_status = Job_Finished; - if(m_opened_for_saving) + if (m_opened_for_saving) { // save the data to the downloadable if we aren't saving to file m_output_file.close(); @@ -72,20 +74,23 @@ void CacheDownload::downloadFinished() } else { - if ( m_output_file.open ( QIODevice::ReadOnly ) ) + if (m_output_file.open(QIODevice::ReadOnly)) { - m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); + m_entry->md5sum = + QCryptographicHash::hash(m_output_file.readAll(), QCryptographicHash::Md5) + .toHex() + .constData(); m_output_file.close(); } } QFileInfo output_file_info(m_target_path); - - + m_entry->etag = m_reply->rawHeader("ETag").constData(); - m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); + m_entry->last_changed_timestamp = + output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); m_entry->stale = false; MMC->metacache()->updateEntry(m_entry); - + m_reply.clear(); emit succeeded(index_within_job); return; @@ -103,9 +108,9 @@ void CacheDownload::downloadFinished() void CacheDownload::downloadReadyRead() { - if(!m_opened_for_saving) + if (!m_opened_for_saving) { - if ( !m_output_file.open ( QIODevice::WriteOnly ) ) + if (!m_output_file.open(QIODevice::WriteOnly)) { /* * Can't open the file... the job failed @@ -118,5 +123,5 @@ void CacheDownload::downloadReadyRead() } QByteArray ba = m_reply->readAll(); md5sum.addData(ba); - m_output_file.write ( ba ); + m_output_file.write(ba); } diff --git a/logic/net/HttpMetaCache.h b/logic/net/HttpMetaCache.h index fac6bec3..daf6c43f 100644 --- a/logic/net/HttpMetaCache.h +++ b/logic/net/HttpMetaCache.h @@ -24,25 +24,28 @@ public: // supply path to the cache index file HttpMetaCache(QString path); ~HttpMetaCache(); - + // get the entry solely from the cache // you probably don't want this, unless you have some specific caching needs. MetaEntryPtr getEntry(QString base, QString resource_path); - + // get the entry from cache and verify that it isn't stale (within reason) - MetaEntryPtr resolveEntry(QString base, QString resource_path, QString expected_etag = QString()); - + MetaEntryPtr resolveEntry(QString base, QString resource_path, + QString expected_etag = QString()); + // add a previously resolved stale entry bool updateEntry(MetaEntryPtr stale_entry); - + void addBase(QString base, QString base_root); - + // (re)start a timer that calls SaveNow later. void SaveEventually(); void Load(); - QString getBasePath ( QString base ); -public slots: + QString getBasePath(QString base); +public +slots: void SaveNow(); + private: // create a new stale entry, given the parameters MetaEntryPtr staleEntry(QString base, QString resource_path);