From 923347729557eed76e4f7e9f6f5f1a79216de0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sat, 26 Oct 2013 19:55:48 +0200 Subject: [PATCH] S3 bucket listing support and network code refactors. * Adds support for listing all objects in an S3 bucket. * Renames a bunch of network related classes (Download->Action) * Net actions now have static constructors --- CMakeLists.txt | 9 +- gui/LegacyModEditDialog.cpp | 4 +- gui/LegacyModEditDialog.h | 4 +- gui/OneSixModEditDialog.cpp | 4 +- gui/mainwindow.cpp | 43 +++--- logic/BaseUpdate.h | 2 +- logic/LegacyUpdate.cpp | 4 +- logic/LegacyUpdate.h | 4 +- logic/OneSixAssets.cpp | 99 +++++-------- logic/OneSixAssets.h | 8 +- logic/OneSixUpdate.cpp | 14 +- logic/OneSixUpdate.h | 6 +- logic/lists/ForgeVersionList.cpp | 15 +- logic/lists/ForgeVersionList.h | 4 +- logic/net/ByteArrayDownload.cpp | 2 +- logic/net/ByteArrayDownload.h | 23 ++-- logic/net/CacheDownload.cpp | 2 +- logic/net/CacheDownload.h | 22 +-- logic/net/FileDownload.cpp | 2 +- logic/net/FileDownload.h | 25 ++-- logic/net/ForgeXzDownload.cpp | 2 +- logic/net/ForgeXzDownload.h | 22 +-- logic/net/{Download.h => NetAction.h} | 9 +- logic/net/{DownloadJob.cpp => NetJob.cpp} | 52 +------ logic/net/{DownloadJob.h => NetJob.h} | 64 +++++---- logic/net/S3ListBucket.cpp | 161 ++++++++++++++++++++++ logic/net/S3ListBucket.h | 42 ++++++ 27 files changed, 405 insertions(+), 243 deletions(-) rename logic/net/{Download.h => NetAction.h} (84%) rename logic/net/{DownloadJob.cpp => NetJob.cpp} (61%) rename logic/net/{DownloadJob.h => NetJob.h} (59%) create mode 100644 logic/net/S3ListBucket.cpp create mode 100644 logic/net/S3ListBucket.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b7dd6ea3..16825be3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -241,7 +241,7 @@ logic/InstanceLauncher.h logic/InstanceLauncher.cpp # network stuffs -logic/net/Download.h +logic/net/NetAction.h logic/net/FileDownload.h logic/net/FileDownload.cpp logic/net/ByteArrayDownload.h @@ -250,12 +250,15 @@ logic/net/CacheDownload.h logic/net/CacheDownload.cpp logic/net/ForgeXzDownload.h logic/net/ForgeXzDownload.cpp -logic/net/DownloadJob.h -logic/net/DownloadJob.cpp +logic/net/NetJob.h +logic/net/NetJob.cpp logic/net/HttpMetaCache.h logic/net/HttpMetaCache.cpp logic/net/LoginTask.h logic/net/LoginTask.cpp +logic/net/S3ListBucket.h +logic/net/S3ListBucket.cpp + # legacy instances logic/LegacyInstance.h diff --git a/gui/LegacyModEditDialog.cpp b/gui/LegacyModEditDialog.cpp index b230193a..a7021bf9 100644 --- a/gui/LegacyModEditDialog.cpp +++ b/gui/LegacyModEditDialog.cpp @@ -218,8 +218,8 @@ void LegacyModEditDialog::on_addForgeBtn_clicked() auto entry = MMC->metacache()->resolveEntry("minecraftforge", forge->filename); if (entry->stale) { - DownloadJob *fjob = new DownloadJob("Forge download"); - fjob->addCacheDownload(forge->universal_url, entry); + NetJob *fjob = new NetJob("Forge download"); + fjob->addNetAction(CacheDownload::make(forge->universal_url, entry)); ProgressDialog dlg(this); dlg.exec(fjob); if (dlg.result() == QDialog::Accepted) diff --git a/gui/LegacyModEditDialog.h b/gui/LegacyModEditDialog.h index fc3ea1e6..d5582aef 100644 --- a/gui/LegacyModEditDialog.h +++ b/gui/LegacyModEditDialog.h @@ -17,7 +17,7 @@ #include #include "logic/LegacyInstance.h" -#include +#include namespace Ui { @@ -74,5 +74,5 @@ private: std::shared_ptr m_jarmods; std::shared_ptr m_texturepacks; LegacyInstance *m_inst; - DownloadJobPtr forgeJob; + NetJobPtr forgeJob; }; diff --git a/gui/OneSixModEditDialog.cpp b/gui/OneSixModEditDialog.cpp index 88738938..54f7289d 100644 --- a/gui/OneSixModEditDialog.cpp +++ b/gui/OneSixModEditDialog.cpp @@ -166,8 +166,8 @@ void OneSixModEditDialog::on_forgeBtn_clicked() auto entry = MMC->metacache()->resolveEntry("minecraftforge", forgeVersion->filename); if (entry->stale) { - DownloadJob *fjob = new DownloadJob("Forge download"); - fjob->addCacheDownload(forgeVersion->installer_url, entry); + NetJob *fjob = new NetJob("Forge download"); + fjob->addNetAction(CacheDownload::make(forgeVersion->installer_url, entry)); ProgressDialog dlg(this); dlg.exec(fjob); if (dlg.result() == QDialog::Accepted) diff --git a/gui/mainwindow.cpp b/gui/mainwindow.cpp index 46fd5bd7..5d3c52b5 100644 --- a/gui/mainwindow.cpp +++ b/gui/mainwindow.cpp @@ -176,7 +176,8 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi assets_downloader = new OneSixAssets(); connect(assets_downloader, SIGNAL(indexStarted()), SLOT(assetsIndexStarted())); connect(assets_downloader, SIGNAL(filesStarted()), SLOT(assetsFilesStarted())); - connect(assets_downloader, SIGNAL(filesProgress(int, int, int)), SLOT(assetsFilesProgress(int, int, int))); + connect(assets_downloader, SIGNAL(filesProgress(int, int, int)), + SLOT(assetsFilesProgress(int, int, int))); connect(assets_downloader, SIGNAL(failed()), SLOT(assetsFailed())); connect(assets_downloader, SIGNAL(finished()), SLOT(assetsFinished())); assets_downloader->start(); @@ -465,8 +466,10 @@ void MainWindow::instanceActivated(QModelIndex index) (BaseInstance *)index.data(InstanceList::InstancePointerRole).value(); bool autoLogin = MMC->settings()->get("AutoLogin").toBool(); - if(autoLogin) doAutoLogin(); - else doLogin(); + if (autoLogin) + doAutoLogin(); + else + doLogin(); } void MainWindow::on_actionLaunchInstance_triggered() @@ -482,15 +485,15 @@ void MainWindow::doAutoLogin() if (!m_selectedInstance) return; - Keyring * k = Keyring::instance(); + Keyring *k = Keyring::instance(); QStringList accounts = k->getStoredAccounts("minecraft"); - if(!accounts.isEmpty()) + if (!accounts.isEmpty()) { QString username = accounts[0]; QString password = k->getPassword("minecraft", username); - if(!password.isEmpty()) + if (!password.isEmpty()) { QLOG_INFO() << "Automatically logging in with stored account: " << username; m_activeInst = m_selectedInstance; @@ -498,7 +501,8 @@ void MainWindow::doAutoLogin() } else { - QLOG_ERROR() << "Auto login set for account, but no password was found: " << username; + QLOG_ERROR() << "Auto login set for account, but no password was found: " + << username; doLogin(tr("Auto login attempted, but no password is stored.")); } } @@ -515,10 +519,8 @@ void MainWindow::doLogin(QString username, QString password) ProgressDialog *tDialog = new ProgressDialog(this); LoginTask *loginTask = new LoginTask(uInfo, tDialog); - connect(loginTask, SIGNAL(succeeded()), SLOT(onLoginComplete()), - Qt::QueuedConnection); - connect(loginTask, SIGNAL(failed(QString)), SLOT(doLogin(QString)), - Qt::QueuedConnection); + connect(loginTask, SIGNAL(succeeded()), SLOT(onLoginComplete()), Qt::QueuedConnection); + connect(loginTask, SIGNAL(failed(QString)), SLOT(doLogin(QString)), Qt::QueuedConnection); tDialog->exec(loginTask); } @@ -573,10 +575,13 @@ void MainWindow::onLoginComplete() delete updateTask; } - auto job = new DownloadJob("Player skin: " + m_activeLogin.player_name); + auto job = new NetJob("Player skin: " + m_activeLogin.player_name); auto meta = MMC->metacache()->resolveEntry("skins", m_activeLogin.player_name + ".png"); - job->addCacheDownload(QUrl("http://skins.minecraft.net/MinecraftSkins/" + m_activeLogin.player_name + ".png"), meta); + auto action = CacheDownload::make( + QUrl("http://skins.minecraft.net/MinecraftSkins/" + m_activeLogin.player_name + ".png"), + meta); + job->addNetAction(action); meta->stale = true; job->start(); @@ -586,7 +591,7 @@ void MainWindow::onLoginComplete() // Add skin mapping QByteArray data; { - if(!listFile.open(QIODevice::ReadWrite)) + if (!listFile.open(QIODevice::ReadWrite)) { QLOG_ERROR() << "Failed to open/make skins list JSON"; return; @@ -601,7 +606,7 @@ void MainWindow::onLoginComplete() QJsonObject mappings = root.value("mappings").toObject(); QJsonArray usernames = mappings.value(m_activeLogin.username).toArray(); - if(!usernames.contains(m_activeLogin.player_name)) + if (!usernames.contains(m_activeLogin.player_name)) { usernames.prepend(m_activeLogin.player_name); mappings[m_activeLogin.username] = usernames; @@ -642,12 +647,11 @@ void MainWindow::launchInstance(BaseInstance *instance, LoginResponse response) this->hide(); } - console = new ConsoleWindow(proc); connect(proc, SIGNAL(log(QString, MessageLevel::Enum)), console, SLOT(write(QString, MessageLevel::Enum))); - connect(proc, SIGNAL(ended(BaseInstance*)), this, SLOT(instanceEnded(BaseInstance*))); + connect(proc, SIGNAL(ended(BaseInstance *)), this, SLOT(instanceEnded(BaseInstance *))); if (instance->settings().get("ShowConsole").toBool()) { @@ -856,7 +860,7 @@ void MainWindow::checkSetDefaultJava() JavaUtils ju; java = ju.GetDefaultJava(); } - if(java) + if (java) MMC->settings()->set("JavaPath", java->path); else MMC->settings()->set("JavaPath", QString("java")); @@ -876,7 +880,8 @@ void MainWindow::assetsFilesStarted() void MainWindow::assetsFilesProgress(int succeeded, int failed, int total) { QString status = tr("Downloading assets: %1 / %2").arg(succeeded + failed).arg(total); - if(failed > 0) status += tr(" (%1 failed)").arg(failed); + if (failed > 0) + status += tr(" (%1 failed)").arg(failed); status += tr("..."); m_statusRight->setText(status); } diff --git a/logic/BaseUpdate.h b/logic/BaseUpdate.h index 6f1e26f3..9ada0770 100644 --- a/logic/BaseUpdate.h +++ b/logic/BaseUpdate.h @@ -19,7 +19,7 @@ #include #include -#include "net/DownloadJob.h" +#include "net/NetJob.h" #include "tasks/Task.h" diff --git a/logic/LegacyUpdate.cpp b/logic/LegacyUpdate.cpp index 66b4bf8a..5120b241 100644 --- a/logic/LegacyUpdate.cpp +++ b/logic/LegacyUpdate.cpp @@ -233,8 +233,8 @@ void LegacyUpdate::jarStart() QString intended_version_id = inst->intendedVersionId(); urlstr += intended_version_id + "/" + intended_version_id + ".jar"; - auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id); - dljob->addFileDownload(QUrl(urlstr), inst->defaultBaseJar()); + auto dljob = new NetJob("Minecraft.jar for version " + intended_version_id); + dljob->addNetAction(FileDownload::make(QUrl(urlstr), inst->defaultBaseJar())); legacyDownloadJob.reset(dljob); connect(dljob, SIGNAL(succeeded()), SLOT(jarFinished())); connect(dljob, SIGNAL(failed()), SLOT(jarFailed())); diff --git a/logic/LegacyUpdate.h b/logic/LegacyUpdate.h index e84ec56a..69560f55 100644 --- a/logic/LegacyUpdate.h +++ b/logic/LegacyUpdate.h @@ -19,7 +19,7 @@ #include #include -#include "net/DownloadJob.h" +#include "net/NetJob.h" #include "tasks/Task.h" #include "BaseUpdate.h" @@ -66,7 +66,7 @@ private: QString lwjglTargetPath; QString lwjglNativesPath; private: - DownloadJobPtr legacyDownloadJob; + NetJobPtr legacyDownloadJob; }; diff --git a/logic/OneSixAssets.cpp b/logic/OneSixAssets.cpp index ff19a86b..375c87f1 100644 --- a/logic/OneSixAssets.cpp +++ b/logic/OneSixAssets.cpp @@ -2,19 +2,11 @@ #include #include #include "OneSixAssets.h" -#include "net/DownloadJob.h" +#include "net/NetJob.h" #include "net/HttpMetaCache.h" +#include "net/S3ListBucket.h" #include "MultiMC.h" -inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) -{ - QDomNodeList elementList = parent.elementsByTagName(tagname); - if (elementList.count()) - return elementList.at(0).toElement(); - else - return QDomElement(); -} - class ThreadedDeleter : public QThread { Q_OBJECT @@ -22,18 +14,18 @@ public: void run() { QLOG_INFO() << "Cleaning up assets folder..."; - QDirIterator iter ( m_base, QDirIterator::Subdirectories ); + QDirIterator iter(m_base, QDirIterator::Subdirectories); int base_length = m_base.length(); - while ( iter.hasNext() ) + while (iter.hasNext()) { QString filename = iter.next(); - QFileInfo current ( filename ); + QFileInfo current(filename); // we keep the dirs... whatever - if ( current.isDir() ) + if (current.isDir()) continue; QString trimmedf = filename; - trimmedf.remove ( 0, base_length + 1 ); - if ( m_whitelist.contains ( trimmedf ) ) + trimmedf.remove(0, base_length + 1); + if (m_whitelist.contains(trimmedf)) { QLOG_TRACE() << trimmedf << " gets to live"; } @@ -41,7 +33,7 @@ public: { // DO NOT TOLERATE JUNK QLOG_TRACE() << trimmedf << " dies"; - QFile f ( filename ); + QFile f(filename); f.remove(); } } @@ -60,71 +52,41 @@ void OneSixAssets::downloadFinished() deleter->start(); } - -void OneSixAssets::fetchXMLFinished() +void OneSixAssets::S3BucketFinished() { - QString prefix ( "http://s3.amazonaws.com/Minecraft.Resources/" ); - QString fprefix ( "assets/" ); + QString prefix("http://s3.amazonaws.com/Minecraft.Resources/"); nuke_whitelist.clear(); emit filesStarted(); auto firstJob = index_job->first(); - QByteArray ba = std::dynamic_pointer_cast(firstJob)->m_data; + auto objectList = std::dynamic_pointer_cast(firstJob)->objects; - QString xmlErrorMsg; - QDomDocument doc; - if ( !doc.setContent ( ba, false, &xmlErrorMsg ) ) - { - QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" << xmlErrorMsg << ba; - emit failed(); - return; - } - //QRegExp etag_match(".*([a-f0-9]{32}).*"); - QDomNodeList contents = doc.elementsByTagName ( "Contents" ); + NetJob *job = new NetJob("Assets"); - DownloadJob *job = new DownloadJob("Assets"); - connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) ); - connect ( job, SIGNAL(failed()), SIGNAL(failed()) ); - connect ( job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int)) ); + connect(job, SIGNAL(succeeded()), SLOT(downloadFinished())); + connect(job, SIGNAL(failed()), SIGNAL(failed())); + connect(job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int))); auto metacache = MMC->metacache(); - - for ( int i = 0; i < contents.length(); i++ ) + + for (auto object: objectList) { - QDomElement element = contents.at ( i ).toElement(); - - if ( element.isNull() ) + // Filter folder keys (zero size) + if (object.size == 0) continue; - QDomElement keyElement = getDomElementByTagName ( element, "Key" ); - QDomElement lastmodElement = getDomElementByTagName ( element, "LastModified" ); - QDomElement etagElement = getDomElementByTagName ( element, "ETag" ); - QDomElement sizeElement = getDomElementByTagName ( element, "Size" ); + nuke_whitelist.append(object.Key); - if ( keyElement.isNull() || lastmodElement.isNull() || etagElement.isNull() || sizeElement.isNull() ) - continue; - - QString keyStr = keyElement.text(); - QString lastModStr = lastmodElement.text(); - QString etagStr = etagElement.text(); - QString sizeStr = sizeElement.text(); - - //Filter folder keys - if ( sizeStr == "0" ) - continue; - - nuke_whitelist.append ( keyStr ); - - auto entry = metacache->resolveEntry("assets", keyStr, etagStr); - if(entry->stale) + auto entry = metacache->resolveEntry("assets", object.Key, object.ETag); + if (entry->stale) { - job->addCacheDownload(QUrl(prefix + keyStr), entry); + job->addNetAction(CacheDownload::make(QUrl(prefix + object.Key), entry)); } } - if(job->size()) + if (job->size()) { - files_job.reset ( job ); + files_job.reset(job); files_job->start(); } else @@ -136,11 +98,12 @@ void OneSixAssets::fetchXMLFinished() void OneSixAssets::start() { - auto job = new DownloadJob("Assets index"); - job->addByteArrayDownload(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" )); - connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) ); + auto job = new NetJob("Assets index"); + job->addNetAction( + S3ListBucket::make(QUrl("http://s3.amazonaws.com/Minecraft.Resources/"))); + connect(job, SIGNAL(succeeded()), SLOT(S3BucketFinished())); emit indexStarted(); - index_job.reset ( job ); + index_job.reset(job); job->start(); } diff --git a/logic/OneSixAssets.h b/logic/OneSixAssets.h index 6c6f2c36..72e8a843 100644 --- a/logic/OneSixAssets.h +++ b/logic/OneSixAssets.h @@ -1,5 +1,5 @@ #pragma once -#include "net/DownloadJob.h" +#include "net/NetJob.h" class Private; class ThreadedDeleter; @@ -15,13 +15,13 @@ signals: void filesProgress(int, int, int); public slots: - void fetchXMLFinished(); + void S3BucketFinished(); void downloadFinished(); public: void start(); private: ThreadedDeleter * deleter; QStringList nuke_whitelist; - DownloadJobPtr index_job; - DownloadJobPtr files_job; + NetJobPtr index_job; + NetJobPtr files_job; }; diff --git a/logic/OneSixUpdate.cpp b/logic/OneSixUpdate.cpp index b5f1d78b..5c421fbf 100644 --- a/logic/OneSixUpdate.cpp +++ b/logic/OneSixUpdate.cpp @@ -72,8 +72,8 @@ void OneSixUpdate::versionFileStart() QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/"); urlstr += targetVersion->descriptor() + "/" + targetVersion->descriptor() + ".json"; - auto job = new DownloadJob("Version index"); - job->addByteArrayDownload(QUrl(urlstr)); + auto job = new NetJob("Version index"); + job->addNetAction(ByteArrayDownload::make(QUrl(urlstr))); specificVersionDownloadJob.reset(job); connect(specificVersionDownloadJob.get(), SIGNAL(succeeded()), SLOT(versionFileFinished())); connect(specificVersionDownloadJob.get(), SIGNAL(failed()), SLOT(versionFileFailed())); @@ -84,7 +84,7 @@ void OneSixUpdate::versionFileStart() void OneSixUpdate::versionFileFinished() { - DownloadPtr DlJob = specificVersionDownloadJob->first(); + NetActionPtr DlJob = specificVersionDownloadJob->first(); OneSixInstance *inst = (OneSixInstance *)m_inst; QString version_id = targetVersion->descriptor(); @@ -154,8 +154,8 @@ void OneSixUpdate::jarlibStart() QString targetstr("versions/"); targetstr += version->id + "/" + version->id + ".jar"; - auto job = new DownloadJob("Libraries for instance " + inst->name()); - job->addFileDownload(QUrl(urlstr), targetstr); + auto job = new NetJob("Libraries for instance " + inst->name()); + job->addNetAction(FileDownload::make(QUrl(urlstr), targetstr)); jarlibDownloadJob.reset(job); auto libs = version->getActiveNativeLibs(); @@ -171,9 +171,9 @@ void OneSixUpdate::jarlibStart() if (entry->stale) { if (lib->hint() == "forge-pack-xz") - jarlibDownloadJob->addForgeXzDownload(download_path, entry); + jarlibDownloadJob->addNetAction(ForgeXzDownload::make(download_path, entry)); else - jarlibDownloadJob->addCacheDownload(download_path, entry); + jarlibDownloadJob->addNetAction(CacheDownload::make(download_path, entry)); } } connect(jarlibDownloadJob.get(), SIGNAL(succeeded()), SLOT(jarlibFinished())); diff --git a/logic/OneSixUpdate.h b/logic/OneSixUpdate.h index ed8dcbcd..7c6fce1b 100644 --- a/logic/OneSixUpdate.h +++ b/logic/OneSixUpdate.h @@ -18,7 +18,7 @@ #include #include #include -#include "net/DownloadJob.h" +#include "net/NetJob.h" #include "tasks/Task.h" #include "BaseUpdate.h" @@ -43,8 +43,8 @@ private slots: void jarlibFailed(); private: - DownloadJobPtr specificVersionDownloadJob; - DownloadJobPtr jarlibDownloadJob; + NetJobPtr specificVersionDownloadJob; + NetJobPtr jarlibDownloadJob; // target version, determined during this task std::shared_ptr targetVersion; diff --git a/logic/lists/ForgeVersionList.cpp b/logic/lists/ForgeVersionList.cpp index 491a43d7..27f567cd 100644 --- a/logic/lists/ForgeVersionList.cpp +++ b/logic/lists/ForgeVersionList.cpp @@ -14,7 +14,7 @@ */ #include "ForgeVersionList.h" -#include +#include #include "MultiMC.h" #include @@ -159,14 +159,14 @@ ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList *vlist) : Task() void ForgeListLoadTask::executeTask() { - auto job = new DownloadJob("Version index"); + auto job = new NetJob("Version index"); // we do not care if the version is stale or not. auto forgeListEntry = MMC->metacache()->resolveEntry("minecraftforge", "list.json"); - + // verify by poking the server. forgeListEntry->stale = true; - - job->addCacheDownload(QUrl(JSON_URL), forgeListEntry); + + job->addNetAction(CacheDownload::make(QUrl(JSON_URL), forgeListEntry)); listJob.reset(job); connect(listJob.get(), SIGNAL(succeeded()), SLOT(list_downloaded())); connect(listJob.get(), SIGNAL(failed()), SLOT(list_failed())); @@ -178,7 +178,7 @@ void ForgeListLoadTask::list_failed() { auto DlJob = listJob->first(); auto reply = DlJob->m_reply; - if(reply) + if (reply) { QLOG_ERROR() << "Getting forge version list failed: " << reply->errorString(); } @@ -193,7 +193,7 @@ void ForgeListLoadTask::list_downloaded() auto DlJob = listJob->first(); auto filename = std::dynamic_pointer_cast(DlJob)->m_target_path; QFile listFile(filename); - if(!listFile.open(QIODevice::ReadOnly)) + if (!listFile.open(QIODevice::ReadOnly)) return; data = listFile.readAll(); DlJob.reset(); @@ -202,7 +202,6 @@ void ForgeListLoadTask::list_downloaded() QJsonParseError jsonError; QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError); - if (jsonError.error != QJsonParseError::NoError) { emitFailed("Error parsing version list JSON:" + jsonError.errorString()); diff --git a/logic/lists/ForgeVersionList.h b/logic/lists/ForgeVersionList.h index 56a207c8..52593f94 100644 --- a/logic/lists/ForgeVersionList.h +++ b/logic/lists/ForgeVersionList.h @@ -23,7 +23,7 @@ #include #include "BaseVersionList.h" #include "logic/tasks/Task.h" -#include "logic/net/DownloadJob.h" +#include "logic/net/NetJob.h" class ForgeVersion; typedef std::shared_ptr ForgeVersionPtr; @@ -104,6 +104,6 @@ slots: void list_failed(); protected: - DownloadJobPtr listJob; + NetJobPtr listJob; ForgeVersionList *m_list; }; diff --git a/logic/net/ByteArrayDownload.cpp b/logic/net/ByteArrayDownload.cpp index ba771eef..25e6d51a 100644 --- a/logic/net/ByteArrayDownload.cpp +++ b/logic/net/ByteArrayDownload.cpp @@ -2,7 +2,7 @@ #include "MultiMC.h" #include -ByteArrayDownload::ByteArrayDownload(QUrl url) : Download() +ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction() { m_url = url; m_status = Job_NotStarted; diff --git a/logic/net/ByteArrayDownload.h b/logic/net/ByteArrayDownload.h index cfc6a8d0..fc32dc04 100644 --- a/logic/net/ByteArrayDownload.h +++ b/logic/net/ByteArrayDownload.h @@ -1,24 +1,29 @@ #pragma once -#include "Download.h" +#include "NetAction.h" -class ByteArrayDownload: public Download +typedef std::shared_ptr ByteArrayDownloadPtr; +class ByteArrayDownload : public NetAction { Q_OBJECT public: ByteArrayDownload(QUrl url); - + static ByteArrayDownloadPtr make(QUrl url) + { + return ByteArrayDownloadPtr(new ByteArrayDownload(url)); + } + public: /// if not saving to file, downloaded data is placed here QByteArray m_data; - -public slots: + +public +slots: virtual void start(); - -protected slots: + +protected +slots: void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); void downloadError(QNetworkReply::NetworkError error); void downloadFinished(); void downloadReadyRead(); }; - -typedef std::shared_ptr ByteArrayDownloadPtr; diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index 309eb345..f8769576 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -8,7 +8,7 @@ #include CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) - : Download(), md5sum(QCryptographicHash::Md5) + : NetAction(), md5sum(QCryptographicHash::Md5) { m_url = url; m_entry = entry; diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h index 295391b1..1e70874c 100644 --- a/logic/net/CacheDownload.h +++ b/logic/net/CacheDownload.h @@ -1,11 +1,12 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include "HttpMetaCache.h" #include #include -class CacheDownload : public Download +typedef std::shared_ptr CacheDownloadPtr; +class CacheDownload : public NetAction { Q_OBJECT public: @@ -18,17 +19,22 @@ public: QFile m_output_file; /// the hash-as-you-download QCryptographicHash md5sum; + public: explicit CacheDownload(QUrl url, MetaEntryPtr entry); - -protected slots: + static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry) + { + return CacheDownloadPtr(new CacheDownload(url, entry)); + } + +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); }; - -typedef std::shared_ptr CacheDownloadPtr; diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp index 3f38b0fa..eefdd4da 100644 --- a/logic/net/FileDownload.cpp +++ b/logic/net/FileDownload.cpp @@ -6,7 +6,7 @@ FileDownload::FileDownload ( QUrl url, QString target_path ) - :Download() + :NetAction() { m_url = url; m_target_path = target_path; diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h index 9abb590d..5f72587f 100644 --- a/logic/net/FileDownload.h +++ b/logic/net/FileDownload.h @@ -1,14 +1,16 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include -class FileDownload : public Download +typedef std::shared_ptr FileDownloadPtr; +class FileDownload : public NetAction { Q_OBJECT public: /// if true, check the md5sum against a provided md5sum - /// also, if a file exists, perform an md5sum first and don't download only if they don't match + /// also, if a file exists, perform an md5sum first and don't download only if they don't + /// match bool m_check_md5; /// the expected md5 checksum QString m_expected_md5; @@ -18,18 +20,21 @@ public: QString m_target_path; /// this is the output file, if any QFile m_output_file; - + public: explicit FileDownload(QUrl url, QString target_path); - -protected slots: + static FileDownloadPtr make(QUrl url, QString target_path) + { + return FileDownloadPtr(new FileDownload(url, target_path)); + } +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); }; - -typedef std::shared_ptr FileDownloadPtr; diff --git a/logic/net/ForgeXzDownload.cpp b/logic/net/ForgeXzDownload.cpp index 0e5287d8..20279d99 100644 --- a/logic/net/ForgeXzDownload.cpp +++ b/logic/net/ForgeXzDownload.cpp @@ -8,7 +8,7 @@ #include ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry) - : Download() + : NetAction() { QString urlstr = url.toString(); urlstr.append(".pack.xz"); diff --git a/logic/net/ForgeXzDownload.h b/logic/net/ForgeXzDownload.h index 5d677947..0b73711e 100644 --- a/logic/net/ForgeXzDownload.h +++ b/logic/net/ForgeXzDownload.h @@ -1,11 +1,12 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include "HttpMetaCache.h" #include #include +typedef std::shared_ptr ForgeXzDownloadPtr; -class ForgeXzDownload : public Download +class ForgeXzDownload : public NetAction { Q_OBJECT public: @@ -19,17 +20,22 @@ public: public: explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry); - -protected slots: + static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry) + { + return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry)); + } + +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); + private: void decompressAndInstall(); }; - -typedef std::shared_ptr ForgeXzDownloadPtr; diff --git a/logic/net/Download.h b/logic/net/NetAction.h similarity index 84% rename from logic/net/Download.h rename to logic/net/NetAction.h index ca4bee9f..b7c922f5 100644 --- a/logic/net/Download.h +++ b/logic/net/NetAction.h @@ -13,14 +13,15 @@ enum JobStatus Job_Failed }; -class Download : public QObject +typedef std::shared_ptr NetActionPtr; +class NetAction : public QObject { Q_OBJECT protected: - explicit Download() : QObject(0) {}; + explicit NetAction() : QObject(0) {}; public: - virtual ~Download() {}; + virtual ~NetAction() {}; public: /// the network reply @@ -50,5 +51,3 @@ protected slots: public slots: virtual void start() = 0; }; - -typedef std::shared_ptr DownloadPtr; diff --git a/logic/net/DownloadJob.cpp b/logic/net/NetJob.cpp similarity index 61% rename from logic/net/DownloadJob.cpp rename to logic/net/NetJob.cpp index 38716a02..c7ca4409 100644 --- a/logic/net/DownloadJob.cpp +++ b/logic/net/NetJob.cpp @@ -1,4 +1,4 @@ -#include "DownloadJob.h" +#include "NetJob.h" #include "pathutils.h" #include "MultiMC.h" #include "FileDownload.h" @@ -7,47 +7,7 @@ #include -ByteArrayDownloadPtr DownloadJob::addByteArrayDownload(QUrl url) -{ - ByteArrayDownloadPtr ptr(new ByteArrayDownload(url)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -FileDownloadPtr DownloadJob::addFileDownload(QUrl url, QString rel_target_path) -{ - FileDownloadPtr ptr(new FileDownload(url, rel_target_path)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -CacheDownloadPtr DownloadJob::addCacheDownload(QUrl url, MetaEntryPtr entry) -{ - CacheDownloadPtr ptr(new CacheDownload(url, entry)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -ForgeXzDownloadPtr DownloadJob::addForgeXzDownload(QUrl url, MetaEntryPtr entry) -{ - ForgeXzDownloadPtr ptr(new ForgeXzDownload(url, entry)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -void DownloadJob::partSucceeded(int index) +void NetJob::partSucceeded(int index) { // do progress. all slots are 1 in size at least auto &slot = parts_progress[index]; @@ -73,7 +33,7 @@ void DownloadJob::partSucceeded(int index) } } -void DownloadJob::partFailed(int index) +void NetJob::partFailed(int index) { auto &slot = parts_progress[index]; if (slot.failures == 3) @@ -97,7 +57,7 @@ void DownloadJob::partFailed(int index) } } -void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) +void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) { auto &slot = parts_progress[index]; @@ -111,7 +71,7 @@ void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTota emit progress(current_progress, total_progress); } -void DownloadJob::start() +void NetJob::start() { QLOG_INFO() << m_job_name.toLocal8Bit() << " started."; for (auto iter : downloads) @@ -124,7 +84,7 @@ void DownloadJob::start() } } -QStringList DownloadJob::getFailedFiles() +QStringList NetJob::getFailedFiles() { QStringList failed; for (auto download : downloads) diff --git a/logic/net/DownloadJob.h b/logic/net/NetJob.h similarity index 59% rename from logic/net/DownloadJob.h rename to logic/net/NetJob.h index cc2a1d59..01d12e60 100644 --- a/logic/net/DownloadJob.h +++ b/logic/net/NetJob.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include "Download.h" +#include "NetAction.h" #include "ByteArrayDownload.h" #include "FileDownload.h" #include "CacheDownload.h" @@ -9,51 +9,57 @@ #include "ForgeXzDownload.h" #include "logic/tasks/ProgressProvider.h" -class DownloadJob; -typedef std::shared_ptr DownloadJobPtr; +class NetJob; +typedef std::shared_ptr NetJobPtr; -/** - * A single file for the downloader/cache to process. - */ -class DownloadJob : public ProgressProvider +class NetJob : public ProgressProvider { Q_OBJECT public: - explicit DownloadJob(QString job_name) - :ProgressProvider(), m_job_name(job_name){}; - - ByteArrayDownloadPtr addByteArrayDownload(QUrl url); - FileDownloadPtr addFileDownload(QUrl url, QString rel_target_path); - CacheDownloadPtr addCacheDownload(QUrl url, MetaEntryPtr entry); - ForgeXzDownloadPtr addForgeXzDownload(QUrl url, MetaEntryPtr entry); - - DownloadPtr operator[](int index) + explicit NetJob(QString job_name) : ProgressProvider(), m_job_name(job_name) {}; + + template + bool addNetAction(T action) + { + NetActionPtr base = std::static_pointer_cast(action); + base->index_within_job = downloads.size(); + downloads.append(action); + parts_progress.append(part_info()); + total_progress++; + return true; + } + + NetActionPtr operator[](int index) { return downloads[index]; - }; - DownloadPtr first() + } + ; + NetActionPtr first() { - if(downloads.size()) + if (downloads.size()) return downloads[0]; - return DownloadPtr(); + return NetActionPtr(); } int size() const { return downloads.size(); } - virtual void getProgress(qint64& current, qint64& total) + virtual void getProgress(qint64 ¤t, qint64 &total) { current = current_progress; total = total_progress; - }; + } + ; virtual QString getStatus() const { return m_job_name; - }; + } + ; virtual bool isRunning() const { return m_running; - }; + } + ; QStringList getFailedFiles(); signals: void started(); @@ -61,12 +67,15 @@ signals: void filesProgress(int, int, int); void succeeded(); void failed(); -public slots: +public +slots: virtual void start(); -private slots: +private +slots: void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal); void partSucceeded(int index); void partFailed(int index); + private: struct part_info { @@ -75,7 +84,7 @@ private: int failures = 0; }; QString m_job_name; - QList downloads; + QList downloads; QList parts_progress; qint64 current_progress = 0; qint64 total_progress = 0; @@ -83,4 +92,3 @@ private: int num_failed = 0; bool m_running = false; }; - diff --git a/logic/net/S3ListBucket.cpp b/logic/net/S3ListBucket.cpp new file mode 100644 index 00000000..643c3224 --- /dev/null +++ b/logic/net/S3ListBucket.cpp @@ -0,0 +1,161 @@ +#include "S3ListBucket.h" +#include "MultiMC.h" +#include +#include +#include +#include + +inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) +{ + QDomNodeList elementList = parent.elementsByTagName(tagname); + if (elementList.count()) + return elementList.at(0).toElement(); + else + return QDomElement(); +} + +S3ListBucket::S3ListBucket(QUrl url) : NetAction() +{ + m_url = url; + m_status = Job_NotStarted; +} + +void S3ListBucket::start() +{ + QUrl finalUrl = m_url; + if (current_marker.size()) + { + QUrlQuery query; + query.addQueryItem("marker", current_marker); + finalUrl.setQuery(query); + } + QNetworkRequest request(finalUrl); + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); + auto worker = MMC->qnam(); + QNetworkReply *rep = worker->get(request); + + m_reply = std::shared_ptr(rep); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); +} + +void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal); +} + +void S3ListBucket::downloadError(QNetworkReply::NetworkError error) +{ + // error happened during download. + QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit() + << "Network error: " << error; + m_status = Job_Failed; +} + +void S3ListBucket::processValidReply() +{ + QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker; + auto readContents = [&](QXmlStreamReader & xml) + { + QString Key, ETag, Size; + while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents")) + { + if (xml.tokenType() == QXmlStreamReader::StartElement) + { + if (xml.name() == "Key") + { + Key = xml.readElementText(); + } + if (xml.name() == "ETag") + { + ETag = xml.readElementText(); + } + if (xml.name() == "Size") + { + Size = xml.readElementText(); + } + } + xml.readNext(); + } + if (xml.error() != QXmlStreamReader::NoError) + return; + objects.append({Key, ETag, Size.toLongLong()}); + }; + + // nothing went wrong... + QString prefix("http://s3.amazonaws.com/Minecraft.Resources/"); + QByteArray ba = m_reply->readAll(); + + QString xmlErrorMsg; + + bool is_truncated = false; + QXmlStreamReader xml(ba); + while (!xml.atEnd() && !xml.hasError()) + { + /* Read next element.*/ + QXmlStreamReader::TokenType token = xml.readNext(); + /* If token is just StartDocument, we'll go to next.*/ + if (token == QXmlStreamReader::StartDocument) + { + continue; + } + if (token == QXmlStreamReader::StartElement) + { + /* If it's named person, we'll dig the information from there.*/ + if (xml.name() == "Contents") + { + readContents(xml); + } + else if (xml.name() == "IsTruncated") + { + is_truncated = (xml.readElementText() == "true"); + } + } + } + if (xml.hasError()) + { + QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" + << xml.errorString() << ba; + emit failed(index_within_job); + return; + } + if(is_truncated) + { + current_marker = objects.last().Key; + bytesSoFar += m_reply->size(); + m_reply.reset(); + start(); + } + else + { + m_status = Job_Finished; + m_reply.reset(); + emit succeeded(index_within_job); + } + return; +} + +void S3ListBucket::downloadFinished() +{ + // if the download succeeded + if (m_status != Job_Failed) + { + processValidReply(); + } + // else the download failed + else + { + m_reply.reset(); + emit failed(index_within_job); + return; + } +} + +void S3ListBucket::downloadReadyRead() +{ + // ~_~ +} diff --git a/logic/net/S3ListBucket.h b/logic/net/S3ListBucket.h new file mode 100644 index 00000000..f054532d --- /dev/null +++ b/logic/net/S3ListBucket.h @@ -0,0 +1,42 @@ +#pragma once +#include "NetAction.h" + +struct S3Object +{ + QString Key; + QString ETag; + qlonglong size; +}; + +typedef std::shared_ptr S3ListBucketPtr; +class S3ListBucket : public NetAction +{ + Q_OBJECT +public: + S3ListBucket(QUrl url); + static S3ListBucketPtr make(QUrl url) + { + return S3ListBucketPtr(new S3ListBucket(url)); + } + +public: + QList objects; + +public +slots: + virtual void start() override; + +protected +slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override; + virtual void downloadError(QNetworkReply::NetworkError error) override; + virtual void downloadFinished() override; + virtual void downloadReadyRead() override; + +private: + void processValidReply(); + +private: + qint64 bytesSoFar = 0; + QString current_marker; +};