0xf8/pollymc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

S3 bucket listing support and network code refactors.

Browse Source 
* Adds support for listing all objects in an S3 bucket.
* Renames a bunch of network related classes (Download->Action)
* Net actions now have static constructors
This commit is contained in:
Petr Mrázek 2013-10-26 19:55:48 +02:00
parent c467ebf132
commit 9233477295
27 changed files with 405 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CMakeLists.txt
View File

@ -241,7 +241,7 @@ logic/InstanceLauncher.h
logic/InstanceLauncher.cpp logic/InstanceLauncher.cpp
# network stuffs # network stuffs
logic/net/Download.h logic/net/NetAction.h
logic/net/FileDownload.h logic/net/FileDownload.h
logic/net/FileDownload.cpp logic/net/FileDownload.cpp
logic/net/ByteArrayDownload.h logic/net/ByteArrayDownload.h
@ -250,12 +250,15 @@ logic/net/CacheDownload.h
logic/net/CacheDownload.cpp logic/net/CacheDownload.cpp
logic/net/ForgeXzDownload.h logic/net/ForgeXzDownload.h
logic/net/ForgeXzDownload.cpp logic/net/ForgeXzDownload.cpp
logic/net/DownloadJob.h logic/net/NetJob.h
logic/net/DownloadJob.cpp logic/net/NetJob.cpp
logic/net/HttpMetaCache.h logic/net/HttpMetaCache.h
logic/net/HttpMetaCache.cpp logic/net/HttpMetaCache.cpp
logic/net/LoginTask.h logic/net/LoginTask.h
logic/net/LoginTask.cpp logic/net/LoginTask.cpp
logic/net/S3ListBucket.h
logic/net/S3ListBucket.cpp
# legacy instances # legacy instances
logic/LegacyInstance.h logic/LegacyInstance.h

4
gui/LegacyModEditDialog.cpp
View File

@ -218,8 +218,8 @@ void LegacyModEditDialog::on_addForgeBtn_clicked()
auto entry = MMC->metacache()->resolveEntry("minecraftforge", forge->filename); auto entry = MMC->metacache()->resolveEntry("minecraftforge", forge->filename);
if (entry->stale) if (entry->stale)
{ {
DownloadJob *fjob = new DownloadJob("Forge download"); NetJob *fjob = new NetJob("Forge download");
fjob->addCacheDownload(forge->universal_url, entry); fjob->addNetAction(CacheDownload::make(forge->universal_url, entry));
ProgressDialog dlg(this); ProgressDialog dlg(this);
dlg.exec(fjob); dlg.exec(fjob);
if (dlg.result() == QDialog::Accepted) if (dlg.result() == QDialog::Accepted)

4
gui/LegacyModEditDialog.h
View File

@ -17,7 +17,7 @@
#include <QDialog> #include <QDialog>
#include "logic/LegacyInstance.h" #include "logic/LegacyInstance.h"
#include <logic/net/DownloadJob.h> #include <logic/net/NetJob.h>
namespace Ui namespace Ui
{ {
@ -74,5 +74,5 @@ private:
std::shared_ptr<ModList> m_jarmods; std::shared_ptr<ModList> m_jarmods;
std::shared_ptr<ModList> m_texturepacks; std::shared_ptr<ModList> m_texturepacks;
LegacyInstance *m_inst; LegacyInstance *m_inst;
DownloadJobPtr forgeJob; NetJobPtr forgeJob;
}; };

4
gui/OneSixModEditDialog.cpp
View File

@ -166,8 +166,8 @@ void OneSixModEditDialog::on_forgeBtn_clicked()
auto entry = MMC->metacache()->resolveEntry("minecraftforge", forgeVersion->filename); auto entry = MMC->metacache()->resolveEntry("minecraftforge", forgeVersion->filename);
if (entry->stale) if (entry->stale)
{ {
DownloadJob *fjob = new DownloadJob("Forge download"); NetJob *fjob = new NetJob("Forge download");
fjob->addCacheDownload(forgeVersion->installer_url, entry); fjob->addNetAction(CacheDownload::make(forgeVersion->installer_url, entry));
ProgressDialog dlg(this); ProgressDialog dlg(this);
dlg.exec(fjob); dlg.exec(fjob);
if (dlg.result() == QDialog::Accepted) if (dlg.result() == QDialog::Accepted)

29
gui/mainwindow.cpp
View File

@ -176,7 +176,8 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi
assets_downloader = new OneSixAssets(); assets_downloader = new OneSixAssets();
connect(assets_downloader, SIGNAL(indexStarted()), SLOT(assetsIndexStarted())); connect(assets_downloader, SIGNAL(indexStarted()), SLOT(assetsIndexStarted()));
connect(assets_downloader, SIGNAL(filesStarted()), SLOT(assetsFilesStarted())); connect(assets_downloader, SIGNAL(filesStarted()), SLOT(assetsFilesStarted()));
connect(assets_downloader, SIGNAL(filesProgress(int, int, int)), SLOT(assetsFilesProgress(int, int, int))); connect(assets_downloader, SIGNAL(filesProgress(int, int, int)),
SLOT(assetsFilesProgress(int, int, int)));
connect(assets_downloader, SIGNAL(failed()), SLOT(assetsFailed())); connect(assets_downloader, SIGNAL(failed()), SLOT(assetsFailed()));
connect(assets_downloader, SIGNAL(finished()), SLOT(assetsFinished())); connect(assets_downloader, SIGNAL(finished()), SLOT(assetsFinished()));
assets_downloader->start(); assets_downloader->start();
@ -465,8 +466,10 @@ void MainWindow::instanceActivated(QModelIndex index)
(BaseInstance *)index.data(InstanceList::InstancePointerRole).value<void *>(); (BaseInstance *)index.data(InstanceList::InstancePointerRole).value<void *>();
bool autoLogin = MMC->settings()->get("AutoLogin").toBool(); bool autoLogin = MMC->settings()->get("AutoLogin").toBool();
if(autoLogin) doAutoLogin(); if (autoLogin)
else doLogin(); doAutoLogin();
else
doLogin();
} }
void MainWindow::on_actionLaunchInstance_triggered() void MainWindow::on_actionLaunchInstance_triggered()
@ -498,7 +501,8 @@ void MainWindow::doAutoLogin()
} }
else else
{ {
QLOG_ERROR() << "Auto login set for account, but no password was found: " << username; QLOG_ERROR() << "Auto login set for account, but no password was found: "
<< username;
doLogin(tr("Auto login attempted, but no password is stored.")); doLogin(tr("Auto login attempted, but no password is stored."));
} }
} }
@ -515,10 +519,8 @@ void MainWindow::doLogin(QString username, QString password)
ProgressDialog *tDialog = new ProgressDialog(this); ProgressDialog *tDialog = new ProgressDialog(this);
LoginTask *loginTask = new LoginTask(uInfo, tDialog); LoginTask *loginTask = new LoginTask(uInfo, tDialog);
connect(loginTask, SIGNAL(succeeded()), SLOT(onLoginComplete()), connect(loginTask, SIGNAL(succeeded()), SLOT(onLoginComplete()), Qt::QueuedConnection);
Qt::QueuedConnection); connect(loginTask, SIGNAL(failed(QString)), SLOT(doLogin(QString)), Qt::QueuedConnection);
connect(loginTask, SIGNAL(failed(QString)), SLOT(doLogin(QString)),
Qt::QueuedConnection);
tDialog->exec(loginTask); tDialog->exec(loginTask);
} }
@ -573,10 +575,13 @@ void MainWindow::onLoginComplete()
delete updateTask; delete updateTask;
} }
auto job = new DownloadJob("Player skin: " + m_activeLogin.player_name); auto job = new NetJob("Player skin: " + m_activeLogin.player_name);
auto meta = MMC->metacache()->resolveEntry("skins", m_activeLogin.player_name + ".png"); auto meta = MMC->metacache()->resolveEntry("skins", m_activeLogin.player_name + ".png");
job->addCacheDownload(QUrl("http://skins.minecraft.net/MinecraftSkins/" + m_activeLogin.player_name + ".png"), meta); auto action = CacheDownload::make(
QUrl("http://skins.minecraft.net/MinecraftSkins/" + m_activeLogin.player_name + ".png"),
meta);
job->addNetAction(action);
meta->stale = true; meta->stale = true;
job->start(); job->start();
@ -642,7 +647,6 @@ void MainWindow::launchInstance(BaseInstance *instance, LoginResponse response)
this->hide(); this->hide();
} }
console = new ConsoleWindow(proc); console = new ConsoleWindow(proc);
connect(proc, SIGNAL(log(QString, MessageLevel::Enum)), console, connect(proc, SIGNAL(log(QString, MessageLevel::Enum)), console,
@ -876,7 +880,8 @@ void MainWindow::assetsFilesStarted()
void MainWindow::assetsFilesProgress(int succeeded, int failed, int total) void MainWindow::assetsFilesProgress(int succeeded, int failed, int total)
{ {
QString status = tr("Downloading assets: %1 / %2").arg(succeeded + failed).arg(total); QString status = tr("Downloading assets: %1 / %2").arg(succeeded + failed).arg(total);
if(failed > 0) status += tr(" (%1 failed)").arg(failed); if (failed > 0)
status += tr(" (%1 failed)").arg(failed);
status += tr("..."); status += tr("...");
m_statusRight->setText(status); m_statusRight->setText(status);
} }

2
logic/BaseUpdate.h
View File

@ -19,7 +19,7 @@
#include <QList> #include <QList>
#include <QUrl> #include <QUrl>
#include "net/DownloadJob.h" #include "net/NetJob.h"
#include "tasks/Task.h" #include "tasks/Task.h"

4
logic/LegacyUpdate.cpp
View File

@ -233,8 +233,8 @@ void LegacyUpdate::jarStart()
QString intended_version_id = inst->intendedVersionId(); QString intended_version_id = inst->intendedVersionId();
urlstr += intended_version_id + "/" + intended_version_id + ".jar"; urlstr += intended_version_id + "/" + intended_version_id + ".jar";
auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id); auto dljob = new NetJob("Minecraft.jar for version " + intended_version_id);
dljob->addFileDownload(QUrl(urlstr), inst->defaultBaseJar()); dljob->addNetAction(FileDownload::make(QUrl(urlstr), inst->defaultBaseJar()));
legacyDownloadJob.reset(dljob); legacyDownloadJob.reset(dljob);
connect(dljob, SIGNAL(succeeded()), SLOT(jarFinished())); connect(dljob, SIGNAL(succeeded()), SLOT(jarFinished()));
connect(dljob, SIGNAL(failed()), SLOT(jarFailed())); connect(dljob, SIGNAL(failed()), SLOT(jarFailed()));

4
logic/LegacyUpdate.h
View File

@ -19,7 +19,7 @@
#include <QList> #include <QList>
#include <QUrl> #include <QUrl>
#include "net/DownloadJob.h" #include "net/NetJob.h"
#include "tasks/Task.h" #include "tasks/Task.h"
#include "BaseUpdate.h" #include "BaseUpdate.h"
@ -66,7 +66,7 @@ private:
QString lwjglTargetPath; QString lwjglTargetPath;
QString lwjglNativesPath; QString lwjglNativesPath;
private: private:
DownloadJobPtr legacyDownloadJob; NetJobPtr legacyDownloadJob;
}; };

67
logic/OneSixAssets.cpp
View File

@ -2,19 +2,11 @@
#include <logger/QsLog.h> #include <logger/QsLog.h>
#include <QtXml/QtXml> #include <QtXml/QtXml>
#include "OneSixAssets.h" #include "OneSixAssets.h"
#include "net/DownloadJob.h" #include "net/NetJob.h"
#include "net/HttpMetaCache.h" #include "net/HttpMetaCache.h"
#include "net/S3ListBucket.h"
#include "MultiMC.h" #include "MultiMC.h"
inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
{
QDomNodeList elementList = parent.elementsByTagName(tagname);
if (elementList.count())
return elementList.at(0).toElement();
else
return QDomElement();
}
class ThreadedDeleter : public QThread class ThreadedDeleter : public QThread
{ {
Q_OBJECT Q_OBJECT
@ -60,66 +52,36 @@ void OneSixAssets::downloadFinished()
deleter->start(); deleter->start();
} }
void OneSixAssets::S3BucketFinished()
void OneSixAssets::fetchXMLFinished()
{ {
QString prefix("http://s3.amazonaws.com/Minecraft.Resources/"); QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
QString fprefix ( "assets/" );
nuke_whitelist.clear(); nuke_whitelist.clear();
emit filesStarted(); emit filesStarted();
auto firstJob = index_job->first(); auto firstJob = index_job->first();
QByteArray ba = std::dynamic_pointer_cast<ByteArrayDownload>(firstJob)->m_data; auto objectList = std::dynamic_pointer_cast<S3ListBucket>(firstJob)->objects;
QString xmlErrorMsg; NetJob *job = new NetJob("Assets");
QDomDocument doc;
if ( !doc.setContent ( ba, false, &xmlErrorMsg ) )
{
QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" << xmlErrorMsg << ba;
emit failed();
return;
}
//QRegExp etag_match(".*([a-f0-9]{32}).*");
QDomNodeList contents = doc.elementsByTagName ( "Contents" );
DownloadJob *job = new DownloadJob("Assets");
connect(job, SIGNAL(succeeded()), SLOT(downloadFinished())); connect(job, SIGNAL(succeeded()), SLOT(downloadFinished()));
connect(job, SIGNAL(failed()), SIGNAL(failed())); connect(job, SIGNAL(failed()), SIGNAL(failed()));
connect(job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int))); connect(job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int)));
auto metacache = MMC->metacache(); auto metacache = MMC->metacache();
for ( int i = 0; i < contents.length(); i++ ) for (auto object: objectList)
{ {
QDomElement element = contents.at ( i ).toElement(); // Filter folder keys (zero size)
if (object.size == 0)
if ( element.isNull() )
continue; continue;
QDomElement keyElement = getDomElementByTagName ( element, "Key" ); nuke_whitelist.append(object.Key);
QDomElement lastmodElement = getDomElementByTagName ( element, "LastModified" );
QDomElement etagElement = getDomElementByTagName ( element, "ETag" );
QDomElement sizeElement = getDomElementByTagName ( element, "Size" );
if ( keyElement.isNull() || lastmodElement.isNull() || etagElement.isNull() || sizeElement.isNull() ) auto entry = metacache->resolveEntry("assets", object.Key, object.ETag);
continue;
QString keyStr = keyElement.text();
QString lastModStr = lastmodElement.text();
QString etagStr = etagElement.text();
QString sizeStr = sizeElement.text();
//Filter folder keys
if ( sizeStr == "0" )
continue;
nuke_whitelist.append ( keyStr );
auto entry = metacache->resolveEntry("assets", keyStr, etagStr);
if (entry->stale) if (entry->stale)
{ {
job->addCacheDownload(QUrl(prefix + keyStr), entry); job->addNetAction(CacheDownload::make(QUrl(prefix + object.Key), entry));
} }
} }
if (job->size()) if (job->size())
@ -136,9 +98,10 @@ void OneSixAssets::fetchXMLFinished()
void OneSixAssets::start() void OneSixAssets::start()
{ {
auto job = new DownloadJob("Assets index"); auto job = new NetJob("Assets index");
job->addByteArrayDownload(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" )); job->addNetAction(
connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) ); S3ListBucket::make(QUrl("http://s3.amazonaws.com/Minecraft.Resources/")));
connect(job, SIGNAL(succeeded()), SLOT(S3BucketFinished()));
emit indexStarted(); emit indexStarted();
index_job.reset(job); index_job.reset(job);
job->start(); job->start();

8
logic/OneSixAssets.h
View File

@ -1,5 +1,5 @@
#pragma once #pragma once
#include "net/DownloadJob.h" #include "net/NetJob.h"
class Private; class Private;
class ThreadedDeleter; class ThreadedDeleter;
@ -15,13 +15,13 @@ signals:
void filesProgress(int, int, int); void filesProgress(int, int, int);
public slots: public slots:
void fetchXMLFinished(); void S3BucketFinished();
void downloadFinished(); void downloadFinished();
public: public:
void start(); void start();
private: private:
ThreadedDeleter * deleter; ThreadedDeleter * deleter;
QStringList nuke_whitelist; QStringList nuke_whitelist;
DownloadJobPtr index_job; NetJobPtr index_job;
DownloadJobPtr files_job; NetJobPtr files_job;
}; };

14
logic/OneSixUpdate.cpp
View File

@ -72,8 +72,8 @@ void OneSixUpdate::versionFileStart()
QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/"); QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/");
urlstr += targetVersion->descriptor() + "/" + targetVersion->descriptor() + ".json"; urlstr += targetVersion->descriptor() + "/" + targetVersion->descriptor() + ".json";
auto job = new DownloadJob("Version index"); auto job = new NetJob("Version index");
job->addByteArrayDownload(QUrl(urlstr)); job->addNetAction(ByteArrayDownload::make(QUrl(urlstr)));
specificVersionDownloadJob.reset(job); specificVersionDownloadJob.reset(job);
connect(specificVersionDownloadJob.get(), SIGNAL(succeeded()), SLOT(versionFileFinished())); connect(specificVersionDownloadJob.get(), SIGNAL(succeeded()), SLOT(versionFileFinished()));
connect(specificVersionDownloadJob.get(), SIGNAL(failed()), SLOT(versionFileFailed())); connect(specificVersionDownloadJob.get(), SIGNAL(failed()), SLOT(versionFileFailed()));
@ -84,7 +84,7 @@ void OneSixUpdate::versionFileStart()
void OneSixUpdate::versionFileFinished() void OneSixUpdate::versionFileFinished()
{ {
DownloadPtr DlJob = specificVersionDownloadJob->first(); NetActionPtr DlJob = specificVersionDownloadJob->first();
OneSixInstance *inst = (OneSixInstance *)m_inst; OneSixInstance *inst = (OneSixInstance *)m_inst;
QString version_id = targetVersion->descriptor(); QString version_id = targetVersion->descriptor();
@ -154,8 +154,8 @@ void OneSixUpdate::jarlibStart()
QString targetstr("versions/"); QString targetstr("versions/");
targetstr += version->id + "/" + version->id + ".jar"; targetstr += version->id + "/" + version->id + ".jar";
auto job = new DownloadJob("Libraries for instance " + inst->name()); auto job = new NetJob("Libraries for instance " + inst->name());
job->addFileDownload(QUrl(urlstr), targetstr); job->addNetAction(FileDownload::make(QUrl(urlstr), targetstr));
jarlibDownloadJob.reset(job); jarlibDownloadJob.reset(job);
auto libs = version->getActiveNativeLibs(); auto libs = version->getActiveNativeLibs();
@ -171,9 +171,9 @@ void OneSixUpdate::jarlibStart()
if (entry->stale) if (entry->stale)
{ {
if (lib->hint() == "forge-pack-xz") if (lib->hint() == "forge-pack-xz")
jarlibDownloadJob->addForgeXzDownload(download_path, entry); jarlibDownloadJob->addNetAction(ForgeXzDownload::make(download_path, entry));
else else
jarlibDownloadJob->addCacheDownload(download_path, entry); jarlibDownloadJob->addNetAction(CacheDownload::make(download_path, entry));
} }
} }
connect(jarlibDownloadJob.get(), SIGNAL(succeeded()), SLOT(jarlibFinished())); connect(jarlibDownloadJob.get(), SIGNAL(succeeded()), SLOT(jarlibFinished()));

6
logic/OneSixUpdate.h
View File

@ -18,7 +18,7 @@
#include <QObject> #include <QObject>
#include <QList> #include <QList>
#include <QUrl> #include <QUrl>
#include "net/DownloadJob.h" #include "net/NetJob.h"
#include "tasks/Task.h" #include "tasks/Task.h"
#include "BaseUpdate.h" #include "BaseUpdate.h"
@ -43,8 +43,8 @@ private slots:
void jarlibFailed(); void jarlibFailed();
private: private:
DownloadJobPtr specificVersionDownloadJob; NetJobPtr specificVersionDownloadJob;
DownloadJobPtr jarlibDownloadJob; NetJobPtr jarlibDownloadJob;
// target version, determined during this task // target version, determined during this task
std::shared_ptr<MinecraftVersion> targetVersion; std::shared_ptr<MinecraftVersion> targetVersion;

7
logic/lists/ForgeVersionList.cpp
View File

@ -14,7 +14,7 @@
*/ */
#include "ForgeVersionList.h" #include "ForgeVersionList.h"
#include <logic/net/DownloadJob.h> #include <logic/net/NetJob.h>
#include "MultiMC.h" #include "MultiMC.h"
#include <QtNetwork> #include <QtNetwork>
@ -159,14 +159,14 @@ ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList *vlist) : Task()
void ForgeListLoadTask::executeTask() void ForgeListLoadTask::executeTask()
{ {
auto job = new DownloadJob("Version index"); auto job = new NetJob("Version index");
// we do not care if the version is stale or not. // we do not care if the version is stale or not.
auto forgeListEntry = MMC->metacache()->resolveEntry("minecraftforge", "list.json"); auto forgeListEntry = MMC->metacache()->resolveEntry("minecraftforge", "list.json");
// verify by poking the server. // verify by poking the server.
forgeListEntry->stale = true; forgeListEntry->stale = true;
job->addCacheDownload(QUrl(JSON_URL), forgeListEntry); job->addNetAction(CacheDownload::make(QUrl(JSON_URL), forgeListEntry));
listJob.reset(job); listJob.reset(job);
connect(listJob.get(), SIGNAL(succeeded()), SLOT(list_downloaded())); connect(listJob.get(), SIGNAL(succeeded()), SLOT(list_downloaded()));
connect(listJob.get(), SIGNAL(failed()), SLOT(list_failed())); connect(listJob.get(), SIGNAL(failed()), SLOT(list_failed()));
@ -202,7 +202,6 @@ void ForgeListLoadTask::list_downloaded()
QJsonParseError jsonError; QJsonParseError jsonError;
QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError); QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError);
if (jsonError.error != QJsonParseError::NoError) if (jsonError.error != QJsonParseError::NoError)
{ {
emitFailed("Error parsing version list JSON:" + jsonError.errorString()); emitFailed("Error parsing version list JSON:" + jsonError.errorString());

4
logic/lists/ForgeVersionList.h
View File

@ -23,7 +23,7 @@
#include <QNetworkReply> #include <QNetworkReply>
#include "BaseVersionList.h" #include "BaseVersionList.h"
#include "logic/tasks/Task.h" #include "logic/tasks/Task.h"
#include "logic/net/DownloadJob.h" #include "logic/net/NetJob.h"
class ForgeVersion; class ForgeVersion;
typedef std::shared_ptr<ForgeVersion> ForgeVersionPtr; typedef std::shared_ptr<ForgeVersion> ForgeVersionPtr;
@ -104,6 +104,6 @@ slots:
void list_failed(); void list_failed();
protected: protected:
DownloadJobPtr listJob; NetJobPtr listJob;
ForgeVersionList *m_list; ForgeVersionList *m_list;
}; };

2
logic/net/ByteArrayDownload.cpp
View File

@ -2,7 +2,7 @@
#include "MultiMC.h" #include "MultiMC.h"
#include <logger/QsLog.h> #include <logger/QsLog.h>
ByteArrayDownload::ByteArrayDownload(QUrl url) : Download() ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction()
{ {
m_url = url; m_url = url;
m_status = Job_NotStarted; m_status = Job_NotStarted;

17
logic/net/ByteArrayDownload.h
View File

@ -1,24 +1,29 @@
#pragma once #pragma once
#include "Download.h" #include "NetAction.h"
class ByteArrayDownload: public Download typedef std::shared_ptr<class ByteArrayDownload> ByteArrayDownloadPtr;
class ByteArrayDownload : public NetAction
{ {
Q_OBJECT Q_OBJECT
public: public:
ByteArrayDownload(QUrl url); ByteArrayDownload(QUrl url);
static ByteArrayDownloadPtr make(QUrl url)
{
return ByteArrayDownloadPtr(new ByteArrayDownload(url));
}
public: public:
/// if not saving to file, downloaded data is placed here /// if not saving to file, downloaded data is placed here
QByteArray m_data; QByteArray m_data;
public slots: public
slots:
virtual void start(); virtual void start();
protected slots: protected
slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
void downloadError(QNetworkReply::NetworkError error); void downloadError(QNetworkReply::NetworkError error);
void downloadFinished(); void downloadFinished();
void downloadReadyRead(); void downloadReadyRead();
}; };
typedef std::shared_ptr<ByteArrayDownload> ByteArrayDownloadPtr;

2
logic/net/CacheDownload.cpp
View File

@ -8,7 +8,7 @@
#include <logger/QsLog.h> #include <logger/QsLog.h>
CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry)
: Download(), md5sum(QCryptographicHash::Md5) : NetAction(), md5sum(QCryptographicHash::Md5)
{ {
m_url = url; m_url = url;
m_entry = entry; m_entry = entry;

18
logic/net/CacheDownload.h
View File

@ -1,11 +1,12 @@
#pragma once #pragma once
#include "Download.h" #include "NetAction.h"
#include "HttpMetaCache.h" #include "HttpMetaCache.h"
#include <QFile> #include <QFile>
#include <qcryptographichash.h> #include <qcryptographichash.h>
class CacheDownload : public Download typedef std::shared_ptr<class CacheDownload> CacheDownloadPtr;
class CacheDownload : public NetAction
{ {
Q_OBJECT Q_OBJECT
public: public:
@ -18,17 +19,22 @@ public:
QFile m_output_file; QFile m_output_file;
/// the hash-as-you-download /// the hash-as-you-download
QCryptographicHash md5sum; QCryptographicHash md5sum;
public: public:
explicit CacheDownload(QUrl url, MetaEntryPtr entry); explicit CacheDownload(QUrl url, MetaEntryPtr entry);
static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry)
{
return CacheDownloadPtr(new CacheDownload(url, entry));
}
protected slots: protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished(); virtual void downloadFinished();
virtual void downloadReadyRead(); virtual void downloadReadyRead();
public slots: public
slots:
virtual void start(); virtual void start();
}; };
typedef std::shared_ptr<CacheDownload> CacheDownloadPtr;

2
logic/net/FileDownload.cpp
View File

@ -6,7 +6,7 @@
FileDownload::FileDownload ( QUrl url, QString target_path ) FileDownload::FileDownload ( QUrl url, QString target_path )
:Download() :NetAction()
{ {
m_url = url; m_url = url;
m_target_path = target_path; m_target_path = target_path;

21
logic/net/FileDownload.h
View File

@ -1,14 +1,16 @@
#pragma once #pragma once
#include "Download.h" #include "NetAction.h"
#include <QFile> #include <QFile>
class FileDownload : public Download typedef std::shared_ptr<class FileDownload> FileDownloadPtr;
class FileDownload : public NetAction
{ {
Q_OBJECT Q_OBJECT
public: public:
/// if true, check the md5sum against a provided md5sum /// if true, check the md5sum against a provided md5sum
/// also, if a file exists, perform an md5sum first and don't download only if they don't match /// also, if a file exists, perform an md5sum first and don't download only if they don't
/// match
bool m_check_md5; bool m_check_md5;
/// the expected md5 checksum /// the expected md5 checksum
QString m_expected_md5; QString m_expected_md5;
@ -21,15 +23,18 @@ public:
public: public:
explicit FileDownload(QUrl url, QString target_path); explicit FileDownload(QUrl url, QString target_path);
static FileDownloadPtr make(QUrl url, QString target_path)
protected slots: {
return FileDownloadPtr(new FileDownload(url, target_path));
}
protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished(); virtual void downloadFinished();
virtual void downloadReadyRead(); virtual void downloadReadyRead();
public slots: public
slots:
virtual void start(); virtual void start();
}; };
typedef std::shared_ptr<FileDownload> FileDownloadPtr;

2
logic/net/ForgeXzDownload.cpp
View File

@ -8,7 +8,7 @@
#include <logger/QsLog.h> #include <logger/QsLog.h>
ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry) ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry)
: Download() : NetAction()
{ {
QString urlstr = url.toString(); QString urlstr = url.toString();
urlstr.append(".pack.xz"); urlstr.append(".pack.xz");

18
logic/net/ForgeXzDownload.h
View File

@ -1,11 +1,12 @@
#pragma once #pragma once
#include "Download.h" #include "NetAction.h"
#include "HttpMetaCache.h" #include "HttpMetaCache.h"
#include <QFile> #include <QFile>
#include <QTemporaryFile> #include <QTemporaryFile>
typedef std::shared_ptr<class ForgeXzDownload> ForgeXzDownloadPtr;
class ForgeXzDownload : public Download class ForgeXzDownload : public NetAction
{ {
Q_OBJECT Q_OBJECT
public: public:
@ -19,17 +20,22 @@ public:
public: public:
explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry); explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry);
static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry)
{
return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry));
}
protected slots: protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished(); virtual void downloadFinished();
virtual void downloadReadyRead(); virtual void downloadReadyRead();
public slots: public
slots:
virtual void start(); virtual void start();
private: private:
void decompressAndInstall(); void decompressAndInstall();
}; };
typedef std::shared_ptr<ForgeXzDownload> ForgeXzDownloadPtr;

9
logic/net/Download.h → logic/net/NetAction.h
View File

@ -13,14 +13,15 @@ enum JobStatus
Job_Failed Job_Failed
}; };
class Download : public QObject typedef std::shared_ptr<class NetAction> NetActionPtr;
class NetAction : public QObject
{ {
Q_OBJECT Q_OBJECT
protected: protected:
explicit Download() : QObject(0) {}; explicit NetAction() : QObject(0) {};
public: public:
virtual ~Download() {}; virtual ~NetAction() {};
public: public:
/// the network reply /// the network reply
@ -50,5 +51,3 @@ protected slots:
public slots: public slots:
virtual void start() = 0; virtual void start() = 0;
}; };
typedef std::shared_ptr<Download> DownloadPtr;

52
logic/net/DownloadJob.cpp → logic/net/NetJob.cpp
View File

@ -1,4 +1,4 @@
#include "DownloadJob.h" #include "NetJob.h"
#include "pathutils.h" #include "pathutils.h"
#include "MultiMC.h" #include "MultiMC.h"
#include "FileDownload.h" #include "FileDownload.h"
@ -7,47 +7,7 @@
#include <logger/QsLog.h> #include <logger/QsLog.h>
ByteArrayDownloadPtr DownloadJob::addByteArrayDownload(QUrl url) void NetJob::partSucceeded(int index)
{
ByteArrayDownloadPtr ptr(new ByteArrayDownload(url));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
parts_progress.append(part_info());
total_progress++;
return ptr;
}
FileDownloadPtr DownloadJob::addFileDownload(QUrl url, QString rel_target_path)
{
FileDownloadPtr ptr(new FileDownload(url, rel_target_path));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
parts_progress.append(part_info());
total_progress++;
return ptr;
}
CacheDownloadPtr DownloadJob::addCacheDownload(QUrl url, MetaEntryPtr entry)
{
CacheDownloadPtr ptr(new CacheDownload(url, entry));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
parts_progress.append(part_info());
total_progress++;
return ptr;
}
ForgeXzDownloadPtr DownloadJob::addForgeXzDownload(QUrl url, MetaEntryPtr entry)
{
ForgeXzDownloadPtr ptr(new ForgeXzDownload(url, entry));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
parts_progress.append(part_info());
total_progress++;
return ptr;
}
void DownloadJob::partSucceeded(int index)
{ {
// do progress. all slots are 1 in size at least // do progress. all slots are 1 in size at least
auto &slot = parts_progress[index]; auto &slot = parts_progress[index];
@ -73,7 +33,7 @@ void DownloadJob::partSucceeded(int index)
} }
} }
void DownloadJob::partFailed(int index) void NetJob::partFailed(int index)
{ {
auto &slot = parts_progress[index]; auto &slot = parts_progress[index];
if (slot.failures == 3) if (slot.failures == 3)
@ -97,7 +57,7 @@ void DownloadJob::partFailed(int index)
} }
} }
void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
{ {
auto &slot = parts_progress[index]; auto &slot = parts_progress[index];
@ -111,7 +71,7 @@ void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTota
emit progress(current_progress, total_progress); emit progress(current_progress, total_progress);
} }
void DownloadJob::start() void NetJob::start()
{ {
QLOG_INFO() << m_job_name.toLocal8Bit() << " started."; QLOG_INFO() << m_job_name.toLocal8Bit() << " started.";
for (auto iter : downloads) for (auto iter : downloads)
@ -124,7 +84,7 @@ void DownloadJob::start()
} }
} }
QStringList DownloadJob::getFailedFiles() QStringList NetJob::getFailedFiles()
{ {
QStringList failed; QStringList failed;
for (auto download : downloads) for (auto download : downloads)

56
logic/net/DownloadJob.h → logic/net/NetJob.h
View File

@ -1,7 +1,7 @@
#pragma once #pragma once
#include <QtNetwork> #include <QtNetwork>
#include <QLabel> #include <QLabel>
#include "Download.h" #include "NetAction.h"
#include "ByteArrayDownload.h" #include "ByteArrayDownload.h"
#include "FileDownload.h" #include "FileDownload.h"
#include "CacheDownload.h" #include "CacheDownload.h"
@ -9,33 +9,36 @@
#include "ForgeXzDownload.h" #include "ForgeXzDownload.h"
#include "logic/tasks/ProgressProvider.h" #include "logic/tasks/ProgressProvider.h"
class DownloadJob; class NetJob;
typedef std::shared_ptr<DownloadJob> DownloadJobPtr; typedef std::shared_ptr<NetJob> NetJobPtr;
/** class NetJob : public ProgressProvider
* A single file for the downloader/cache to process.
*/
class DownloadJob : public ProgressProvider
{ {
Q_OBJECT Q_OBJECT
public: public:
explicit DownloadJob(QString job_name) explicit NetJob(QString job_name) : ProgressProvider(), m_job_name(job_name) {};
:ProgressProvider(), m_job_name(job_name){};
ByteArrayDownloadPtr addByteArrayDownload(QUrl url); template <typename T>
FileDownloadPtr addFileDownload(QUrl url, QString rel_target_path); bool addNetAction(T action)
CacheDownloadPtr addCacheDownload(QUrl url, MetaEntryPtr entry); {
ForgeXzDownloadPtr addForgeXzDownload(QUrl url, MetaEntryPtr entry); NetActionPtr base = std::static_pointer_cast<NetAction>(action);
base->index_within_job = downloads.size();
downloads.append(action);
parts_progress.append(part_info());
total_progress++;
return true;
}
DownloadPtr operator[](int index) NetActionPtr operator[](int index)
{ {
return downloads[index]; return downloads[index];
}; }
DownloadPtr first() ;
NetActionPtr first()
{ {
if (downloads.size()) if (downloads.size())
return downloads[0]; return downloads[0];
return DownloadPtr(); return NetActionPtr();
} }
int size() const int size() const
{ {
@ -45,15 +48,18 @@ public:
{ {
current = current_progress; current = current_progress;
total = total_progress; total = total_progress;
}; }
;
virtual QString getStatus() const virtual QString getStatus() const
{ {
return m_job_name; return m_job_name;
}; }
;
virtual bool isRunning() const virtual bool isRunning() const
{ {
return m_running; return m_running;
}; }
;
QStringList getFailedFiles(); QStringList getFailedFiles();
signals: signals:
void started(); void started();
@ -61,12 +67,15 @@ signals:
void filesProgress(int, int, int); void filesProgress(int, int, int);
void succeeded(); void succeeded();
void failed(); void failed();
public slots: public
slots:
virtual void start(); virtual void start();
private slots: private
slots:
void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal); void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);
void partSucceeded(int index); void partSucceeded(int index);
void partFailed(int index); void partFailed(int index);
private: private:
struct part_info struct part_info
{ {
@ -75,7 +84,7 @@ private:
int failures = 0; int failures = 0;
}; };
QString m_job_name; QString m_job_name;
QList<DownloadPtr> downloads; QList<NetActionPtr> downloads;
QList<part_info> parts_progress; QList<part_info> parts_progress;
qint64 current_progress = 0; qint64 current_progress = 0;
qint64 total_progress = 0; qint64 total_progress = 0;
@ -83,4 +92,3 @@ private:
int num_failed = 0; int num_failed = 0;
bool m_running = false; bool m_running = false;
}; };

161
logic/net/S3ListBucket.cpp Normal file
View File

@ -0,0 +1,161 @@
#include "S3ListBucket.h"
#include "MultiMC.h"
#include <logger/QsLog.h>
#include <QUrlQuery>
#include <qxmlstream.h>
#include <QDomDocument>
inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
{
QDomNodeList elementList = parent.elementsByTagName(tagname);
if (elementList.count())
return elementList.at(0).toElement();
else
return QDomElement();
}
S3ListBucket::S3ListBucket(QUrl url) : NetAction()
{
m_url = url;
m_status = Job_NotStarted;
}
void S3ListBucket::start()
{
QUrl finalUrl = m_url;
if (current_marker.size())
{
QUrlQuery query;
query.addQueryItem("marker", current_marker);
finalUrl.setQuery(query);
}
QNetworkRequest request(finalUrl);
request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
auto worker = MMC->qnam();
QNetworkReply *rep = worker->get(request);
m_reply = std::shared_ptr<QNetworkReply>(rep);
connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
SLOT(downloadProgress(qint64, qint64)));
connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
}
void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal);
}
void S3ListBucket::downloadError(QNetworkReply::NetworkError error)
{
// error happened during download.
QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit()
<< "Network error: " << error;
m_status = Job_Failed;
}
void S3ListBucket::processValidReply()
{
QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker;
auto readContents = [&](QXmlStreamReader & xml)
{
QString Key, ETag, Size;
while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents"))
{
if (xml.tokenType() == QXmlStreamReader::StartElement)
{
if (xml.name() == "Key")
{
Key = xml.readElementText();
}
if (xml.name() == "ETag")
{
ETag = xml.readElementText();
}
if (xml.name() == "Size")
{
Size = xml.readElementText();
}
}
xml.readNext();
}
if (xml.error() != QXmlStreamReader::NoError)
return;
objects.append({Key, ETag, Size.toLongLong()});
};
// nothing went wrong...
QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
QByteArray ba = m_reply->readAll();
QString xmlErrorMsg;
bool is_truncated = false;
QXmlStreamReader xml(ba);
while (!xml.atEnd() && !xml.hasError())
{
/* Read next element.*/
QXmlStreamReader::TokenType token = xml.readNext();
/* If token is just StartDocument, we'll go to next.*/
if (token == QXmlStreamReader::StartDocument)
{
continue;
}
if (token == QXmlStreamReader::StartElement)
{
/* If it's named person, we'll dig the information from there.*/
if (xml.name() == "Contents")
{
readContents(xml);
}
else if (xml.name() == "IsTruncated")
{
is_truncated = (xml.readElementText() == "true");
}
}
}
if (xml.hasError())
{
QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:"
<< xml.errorString() << ba;
emit failed(index_within_job);
return;
}
if(is_truncated)
{
current_marker = objects.last().Key;
bytesSoFar += m_reply->size();
m_reply.reset();
start();
}
else
{
m_status = Job_Finished;
m_reply.reset();
emit succeeded(index_within_job);
}
return;
}
void S3ListBucket::downloadFinished()
{
// if the download succeeded
if (m_status != Job_Failed)
{
processValidReply();
}
// else the download failed
else
{
m_reply.reset();
emit failed(index_within_job);
return;
}
}
void S3ListBucket::downloadReadyRead()
{
// ~_~
}

42
logic/net/S3ListBucket.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
#include "NetAction.h"
struct S3Object
{
QString Key;
QString ETag;
qlonglong size;
};
typedef std::shared_ptr<class S3ListBucket> S3ListBucketPtr;
class S3ListBucket : public NetAction
{
Q_OBJECT
public:
S3ListBucket(QUrl url);
static S3ListBucketPtr make(QUrl url)
{
return S3ListBucketPtr(new S3ListBucket(url));
}
public:
QList<S3Object> objects;
public
slots:
virtual void start() override;
protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
virtual void downloadError(QNetworkReply::NetworkError error) override;
virtual void downloadFinished() override;
virtual void downloadReadyRead() override;
private:
void processValidReply();
private:
qint64 bytesSoFar = 0;
QString current_marker;
};