Speed up the downloads. Massively.

This commit is contained in:
Petr Mrázek 2013-09-02 00:25:40 +02:00
parent 78e278c1e3
commit f8e8414d54
18 changed files with 264 additions and 321 deletions

View File

@ -182,8 +182,8 @@ logic/Mod.h
logic/ModList.h logic/ModList.h
# network stuffs # network stuffs
logic/net/Job.h
logic/net/DownloadJob.h logic/net/DownloadJob.h
logic/net/JobQueue.h
logic/net/NetWorker.h logic/net/NetWorker.h
# legacy instances # legacy instances

View File

@ -42,7 +42,6 @@ protected slots:
void updateDownloadProgress(qint64 current, qint64 total); void updateDownloadProgress(qint64 current, qint64 total);
protected: protected:
JobListQueue download_queue;
BaseInstance *m_inst; BaseInstance *m_inst;
}; };

View File

@ -71,7 +71,7 @@ MinecraftProcess* LegacyInstance::prepareForLaunch(QString user, QString session
// launcher arguments // launcher arguments
args << QString("-Xms%1m").arg(settings().get("MinMemAlloc").toInt()); args << QString("-Xms%1m").arg(settings().get("MinMemAlloc").toInt());
args << QString("-Xmx%1m").arg(settings().get("MaxMemAlloc").toInt()); args << QString("-Xmx%1m").arg(settings().get("MaxMemAlloc").toInt());
args << QString("-XX:MaxPermSize=%1m").arg(settings().get("PermGen").toInt()); args << QString("-XX:PermSize=%1m").arg(settings().get("PermGen").toInt());
args << "-jar" << LAUNCHER_FILE; args << "-jar" << LAUNCHER_FILE;
args << user; args << user;
args << session; args << session;

View File

@ -52,7 +52,7 @@ void LegacyUpdate::lwjglStart()
QString url = version->url(); QString url = version->url();
QUrl realUrl(url); QUrl realUrl(url);
QString hostname = realUrl.host(); QString hostname = realUrl.host();
auto &worker = NetWorker::spawn(); auto &worker = NetWorker::qnam();
QNetworkRequest req(realUrl); QNetworkRequest req(realUrl);
req.setRawHeader("Host", hostname.toLatin1()); req.setRawHeader("Host", hostname.toLatin1());
req.setHeader(QNetworkRequest::UserAgentHeader, "Wget/1.14 (linux-gnu)"); req.setHeader(QNetworkRequest::UserAgentHeader, "Wget/1.14 (linux-gnu)");
@ -77,7 +77,7 @@ void LegacyUpdate::lwjglFinished(QNetworkReply* reply)
"\nSometimes you have to wait a bit if you download many LWJGL versions in a row. YMMV"); "\nSometimes you have to wait a bit if you download many LWJGL versions in a row. YMMV");
return; return;
} }
auto &worker = NetWorker::spawn(); auto &worker = NetWorker::qnam();
//Here i check if there is a cookie for me in the reply and extract it //Here i check if there is a cookie for me in the reply and extract it
QList<QNetworkCookie> cookies = qvariant_cast<QList<QNetworkCookie>>(reply->header(QNetworkRequest::SetCookieHeader)); QList<QNetworkCookie> cookies = qvariant_cast<QList<QNetworkCookie>>(reply->header(QNetworkRequest::SetCookieHeader));
if(cookies.count() != 0) if(cookies.count() != 0)
@ -90,7 +90,6 @@ void LegacyUpdate::lwjglFinished(QNetworkReply* reply)
QVariant newLoc = reply->header(QNetworkRequest::LocationHeader); QVariant newLoc = reply->header(QNetworkRequest::LocationHeader);
if(newLoc.isValid()) if(newLoc.isValid())
{ {
auto &worker = NetWorker::spawn();
QString redirectedTo = reply->header(QNetworkRequest::LocationHeader).toString(); QString redirectedTo = reply->header(QNetworkRequest::LocationHeader).toString();
QUrl realUrl(redirectedTo); QUrl realUrl(redirectedTo);
QString hostname = realUrl.host(); QString hostname = realUrl.host();
@ -228,14 +227,11 @@ void LegacyUpdate::jarStart()
QString intended_version_id = inst->intendedVersionId(); QString intended_version_id = inst->intendedVersionId();
urlstr += intended_version_id + "/" + intended_version_id + ".jar"; urlstr += intended_version_id + "/" + intended_version_id + ".jar";
auto dljob = DownloadJob::create(QUrl(urlstr), inst->defaultBaseJar()); legacyDownloadJob.reset(new DownloadJob(QUrl(urlstr), inst->defaultBaseJar()));
legacyDownloadJob.reset(new JobList());
legacyDownloadJob->add(dljob);
connect(legacyDownloadJob.data(), SIGNAL(finished()), SLOT(jarFinished())); connect(legacyDownloadJob.data(), SIGNAL(finished()), SLOT(jarFinished()));
connect(legacyDownloadJob.data(), SIGNAL(failed()), SLOT(jarFailed())); connect(legacyDownloadJob.data(), SIGNAL(failed()), SLOT(jarFailed()));
connect(legacyDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64))); connect(legacyDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
download_queue.enqueue(legacyDownloadJob); legacyDownloadJob->start();
} }
void LegacyUpdate::jarFinished() void LegacyUpdate::jarFinished()

View File

@ -66,8 +66,7 @@ private:
QString lwjglTargetPath; QString lwjglTargetPath;
QString lwjglNativesPath; QString lwjglNativesPath;
private: private:
JobListPtr legacyDownloadJob; DownloadJobPtr legacyDownloadJob;
JobListQueue download_queue;
}; };

View File

@ -19,19 +19,18 @@ class ThreadedDeleter : public QThread
public: public:
void run() void run()
{ {
QDirIterator iter(m_base, QDirIterator::Subdirectories); QDirIterator iter ( m_base, QDirIterator::Subdirectories );
QStringList nuke_list;
int base_length = m_base.length(); int base_length = m_base.length();
while (iter.hasNext()) while ( iter.hasNext() )
{ {
QString filename = iter.next(); QString filename = iter.next();
QFileInfo current(filename); QFileInfo current ( filename );
// we keep the dirs... whatever // we keep the dirs... whatever
if(current.isDir()) if ( current.isDir() )
continue; continue;
QString trimmedf = filename; QString trimmedf = filename;
trimmedf.remove(0, base_length + 1); trimmedf.remove ( 0, base_length + 1 );
if(m_whitelist.contains(trimmedf)) if ( m_whitelist.contains ( trimmedf ) )
{ {
// qDebug() << trimmedf << " gets to live"; // qDebug() << trimmedf << " gets to live";
} }
@ -39,66 +38,47 @@ public:
{ {
// DO NOT TOLERATE JUNK // DO NOT TOLERATE JUNK
// qDebug() << trimmedf << " dies"; // qDebug() << trimmedf << " dies";
QFile f (filename); QFile f ( filename );
f.remove(); f.remove();
} }
} }
}; }
QString m_base; QString m_base;
QStringList m_whitelist; QStringList m_whitelist;
}; };
class NukeAndPaveJob: public Job void OneSixAssets::downloadFinished()
{ {
Q_OBJECT deleter = new ThreadedDeleter();
public: QDir dir("assets");
deleter->m_base = dir.absolutePath();
explicit NukeAndPaveJob(QString base, QStringList whitelist) deleter->m_whitelist = nuke_whitelist;
:Job() connect(deleter, SIGNAL(finished()), SIGNAL(finished()));
{ deleter->start();
QDir dir(base); }
deleterThread.m_base = dir.absolutePath();
deleterThread.m_whitelist = whitelist;
};
public slots:
virtual void start()
{
connect(&deleterThread, SIGNAL(finished()), SLOT(threadFinished()));
deleterThread.start();
};
void threadFinished()
{
emit finish();
}
private:
ThreadedDeleter deleterThread;
};
void OneSixAssets::fetchXMLFinished()
void OneSixAssets::fetchFinished()
{ {
QString prefix ( "http://s3.amazonaws.com/Minecraft.Resources/" ); QString prefix ( "http://s3.amazonaws.com/Minecraft.Resources/" );
QString fprefix ( "assets/" ); QString fprefix ( "assets/" );
QStringList nuke_whitelist; nuke_whitelist.clear();
JobPtr firstJob = index_job->getFirstJob(); auto firstJob = index_job->first();
auto DlJob = firstJob.dynamicCast<DownloadJob>(); QByteArray ba = firstJob->m_data;
QByteArray ba = DlJob->m_data;
QString xmlErrorMsg; QString xmlErrorMsg;
QDomDocument doc; QDomDocument doc;
if ( !doc.setContent ( ba, false, &xmlErrorMsg ) ) if ( !doc.setContent ( ba, false, &xmlErrorMsg ) )
{ {
qDebug() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" << qDebug() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" << xmlErrorMsg << ba;
xmlErrorMsg << ba;
} }
//QRegExp etag_match(".*([a-f0-9]{32}).*"); //QRegExp etag_match(".*([a-f0-9]{32}).*");
QDomNodeList contents = doc.elementsByTagName ( "Contents" ); QDomNodeList contents = doc.elementsByTagName ( "Contents" );
JobList *job = new JobList(); DownloadJob *job = new DownloadJob();
connect ( job, SIGNAL ( finished() ), SIGNAL(finished()) ); connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) );
connect ( job, SIGNAL ( failed() ), SIGNAL(failed()) ); connect ( job, SIGNAL(failed()), SIGNAL(failed()) );
for ( int i = 0; i < contents.length(); i++ ) for ( int i = 0; i < contents.length(); i++ )
{ {
@ -138,25 +118,28 @@ void OneSixAssets::fetchFinished()
QString trimmedEtag = etagStr.remove ( '"' ); QString trimmedEtag = etagStr.remove ( '"' );
nuke_whitelist.append ( keyStr ); nuke_whitelist.append ( keyStr );
if(trimmedEtag != client_etag) if(trimmedEtag != client_etag)
job->add ( DownloadJob::create ( QUrl ( prefix + keyStr ), filename ) ); {
job->add ( QUrl ( prefix + keyStr ), filename );
}
}
if(job->size())
{
files_job.reset ( job );
files_job->start();
}
else
{
delete job;
emit finished();
} }
job->add ( JobPtr ( new NukeAndPaveJob ( fprefix, nuke_whitelist ) ) );
files_job.reset ( job );
dl.enqueue ( files_job );
}
void OneSixAssets::fetchStarted()
{
qDebug() << "Started downloading!";
} }
void OneSixAssets::start() void OneSixAssets::start()
{ {
JobList *job = new JobList(); DownloadJob * job = new DownloadJob(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
job->add ( DownloadJob::create ( QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ) ) ); connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) );
connect ( job, SIGNAL ( finished() ), SLOT ( fetchFinished() ) );
connect ( job, SIGNAL ( started() ), SLOT ( fetchStarted() ) );
index_job.reset ( job ); index_job.reset ( job );
dl.enqueue ( index_job ); job->start();
} }
#include "OneSixAssets.moc"
#include "OneSixAssets.moc"

View File

@ -2,6 +2,7 @@
#include "net/DownloadJob.h" #include "net/DownloadJob.h"
class Private; class Private;
class ThreadedDeleter;
class OneSixAssets : public QObject class OneSixAssets : public QObject
{ {
@ -11,12 +12,13 @@ signals:
void finished(); void finished();
public slots: public slots:
void fetchFinished(); void fetchXMLFinished();
void fetchStarted(); void downloadFinished();
public: public:
void start(); void start();
private: private:
JobListQueue dl; ThreadedDeleter * deleter;
JobListPtr index_job; QStringList nuke_whitelist;
JobListPtr files_job; DownloadJobPtr index_job;
DownloadJobPtr files_job;
}; };

View File

@ -114,7 +114,7 @@ MinecraftProcess* OneSixInstance::prepareForLaunch ( QString user, QString sessi
args.append(Util::Commandline::splitArgs(settings().get("JvmArgs").toString())); args.append(Util::Commandline::splitArgs(settings().get("JvmArgs").toString()));
args << QString("-Xms%1m").arg(settings().get("MinMemAlloc").toInt()); args << QString("-Xms%1m").arg(settings().get("MinMemAlloc").toInt());
args << QString("-Xmx%1m").arg(settings().get("MaxMemAlloc").toInt()); args << QString("-Xmx%1m").arg(settings().get("MaxMemAlloc").toInt());
args << QString("-XX:MaxPermSize=%1m").arg(settings().get("PermGen").toInt()); args << QString("-XX:PermSize=%1m").arg(settings().get("PermGen").toInt());
QDir natives_dir(natives_dir_raw); QDir natives_dir(natives_dir_raw);
args << QString("-Djava.library.path=%1").arg( natives_dir.absolutePath() ); args << QString("-Djava.library.path=%1").arg( natives_dir.absolutePath() );
QString classPath; QString classPath;

View File

@ -72,19 +72,16 @@ void OneSixUpdate::versionFileStart()
QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/"); QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/");
urlstr += targetVersion->descriptor + "/" + targetVersion->descriptor + ".json"; urlstr += targetVersion->descriptor + "/" + targetVersion->descriptor + ".json";
auto dljob = DownloadJob::create(QUrl(urlstr)); specificVersionDownloadJob.reset(new DownloadJob(QUrl(urlstr)));
specificVersionDownloadJob.reset(new JobList()); connect(specificVersionDownloadJob.data(), SIGNAL(succeeded()), SLOT(versionFileFinished()));
specificVersionDownloadJob->add(dljob);
connect(specificVersionDownloadJob.data(), SIGNAL(finished()), SLOT(versionFileFinished()));
connect(specificVersionDownloadJob.data(), SIGNAL(failed()), SLOT(versionFileFailed())); connect(specificVersionDownloadJob.data(), SIGNAL(failed()), SLOT(versionFileFailed()));
connect(specificVersionDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64))); connect(specificVersionDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
download_queue.enqueue(specificVersionDownloadJob); specificVersionDownloadJob->start();
} }
void OneSixUpdate::versionFileFinished() void OneSixUpdate::versionFileFinished()
{ {
JobPtr firstJob = specificVersionDownloadJob->getFirstJob(); DownloadPtr DlJob = specificVersionDownloadJob->first();
auto DlJob = firstJob.dynamicCast<DownloadJob>();
QString version_id = targetVersion->descriptor; QString version_id = targetVersion->descriptor;
QString inst_dir = m_inst->instanceRoot(); QString inst_dir = m_inst->instanceRoot();
@ -137,9 +134,7 @@ void OneSixUpdate::jarlibStart()
QString targetstr ("versions/"); QString targetstr ("versions/");
targetstr += version->id + "/" + version->id + ".jar"; targetstr += version->id + "/" + version->id + ".jar";
auto dljob = DownloadJob::create(QUrl(urlstr), targetstr); jarlibDownloadJob.reset(new DownloadJob(QUrl(urlstr), targetstr));
jarlibDownloadJob.reset(new JobList());
jarlibDownloadJob->add(dljob);
auto libs = version->getActiveNativeLibs(); auto libs = version->getActiveNativeLibs();
libs.append(version->getActiveNormalLibs()); libs.append(version->getActiveNormalLibs());
@ -148,13 +143,13 @@ void OneSixUpdate::jarlibStart()
{ {
QString download_path = lib->downloadPath(); QString download_path = lib->downloadPath();
QString storage_path = "libraries/" + lib->storagePath(); QString storage_path = "libraries/" + lib->storagePath();
jarlibDownloadJob->add(DownloadJob::create(download_path, storage_path)); jarlibDownloadJob->add(download_path, storage_path);
} }
connect(jarlibDownloadJob.data(), SIGNAL(finished()), SLOT(jarlibFinished())); connect(jarlibDownloadJob.data(), SIGNAL(succeeded()), SLOT(jarlibFinished()));
connect(jarlibDownloadJob.data(), SIGNAL(failed()), SLOT(jarlibFailed())); connect(jarlibDownloadJob.data(), SIGNAL(failed()), SLOT(jarlibFailed()));
connect(jarlibDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64))); connect(jarlibDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
download_queue.enqueue(jarlibDownloadJob); jarlibDownloadJob->start();
} }
void OneSixUpdate::jarlibFinished() void OneSixUpdate::jarlibFinished()

View File

@ -43,9 +43,8 @@ private slots:
void jarlibFailed(); void jarlibFailed();
private: private:
JobListPtr specificVersionDownloadJob; DownloadJobPtr specificVersionDownloadJob;
JobListPtr jarlibDownloadJob; DownloadJobPtr jarlibDownloadJob;
JobListQueue download_queue;
// target version, determined during this task // target version, determined during this task
QSharedPointer<MinecraftVersion> targetVersion; QSharedPointer<MinecraftVersion> targetVersion;

View File

@ -91,7 +91,7 @@ void LWJGLVersionList::loadList()
Q_ASSERT_X(!m_loading, "loadList", "list is already loading (m_loading is true)"); Q_ASSERT_X(!m_loading, "loadList", "list is already loading (m_loading is true)");
setLoading(true); setLoading(true);
auto & worker = NetWorker::spawn(); auto & worker = NetWorker::qnam();
reply = worker.get(QNetworkRequest(QUrl(RSS_URL))); reply = worker.get(QNetworkRequest(QUrl(RSS_URL)));
connect(reply, SIGNAL(finished()), SLOT(netRequestComplete())); connect(reply, SIGNAL(finished()), SLOT(netRequestComplete()));
} }

View File

@ -151,7 +151,7 @@ MCVListLoadTask::~MCVListLoadTask()
void MCVListLoadTask::executeTask() void MCVListLoadTask::executeTask()
{ {
setStatus("Loading instance version list..."); setStatus("Loading instance version list...");
auto & worker = NetWorker::spawn(); auto & worker = NetWorker::qnam();
vlistReply = worker.get(QNetworkRequest(QUrl(QString(MCVLIST_URLBASE) + "versions.json"))); vlistReply = worker.get(QNetworkRequest(QUrl(QString(MCVLIST_URLBASE) + "versions.json")));
connect(vlistReply, SIGNAL(finished()), this, SLOT(list_downloaded())); connect(vlistReply, SIGNAL(finished()), this, SLOT(list_downloaded()));
} }

View File

@ -2,9 +2,7 @@
#include "pathutils.h" #include "pathutils.h"
#include "NetWorker.h" #include "NetWorker.h"
DownloadJob::DownloadJob (QUrl url, Download::Download (QUrl url, QString target_path, QString expected_md5 )
QString target_path,
QString expected_md5 )
:Job() :Job()
{ {
m_url = url; m_url = url;
@ -17,14 +15,7 @@ DownloadJob::DownloadJob (QUrl url,
m_opened_for_saving = false; m_opened_for_saving = false;
} }
JobPtr DownloadJob::create (QUrl url, void Download::start()
QString target_path,
QString expected_md5 )
{
return JobPtr ( new DownloadJob ( url, target_path, expected_md5 ) );
}
void DownloadJob::start()
{ {
if ( m_save_to_file ) if ( m_save_to_file )
{ {
@ -40,7 +31,7 @@ void DownloadJob::start()
if ( m_check_md5 && hash == m_expected_md5 ) if ( m_check_md5 && hash == m_expected_md5 )
{ {
qDebug() << "Skipping " << m_url.toString() << ": md5 match."; qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
emit finish(); emit succeeded(index_within_job);
return; return;
} }
else else
@ -50,7 +41,7 @@ void DownloadJob::start()
} }
if(!ensureFilePathExists(filename)) if(!ensureFilePathExists(filename))
{ {
emit fail(); emit failed(index_within_job);
return; return;
} }
} }
@ -58,7 +49,7 @@ void DownloadJob::start()
QNetworkRequest request ( m_url ); QNetworkRequest request ( m_url );
request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1());
auto &worker = NetWorker::spawn(); auto &worker = NetWorker::qnam();
QNetworkReply * rep = worker.get ( request ); QNetworkReply * rep = worker.get ( request );
m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater ); m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
@ -68,19 +59,19 @@ void DownloadJob::start()
connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
} }
void DownloadJob::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
{ {
emit progress ( bytesReceived, bytesTotal ); emit progress (index_within_job, bytesReceived, bytesTotal );
} }
void DownloadJob::downloadError ( QNetworkReply::NetworkError error ) void Download::downloadError ( QNetworkReply::NetworkError error )
{ {
// error happened during download. // error happened during download.
// TODO: log the reason why // TODO: log the reason why
m_status = Job_Failed; m_status = Job_Failed;
} }
void DownloadJob::downloadFinished() void Download::downloadFinished()
{ {
// if the download succeeded // if the download succeeded
if ( m_status != Job_Failed ) if ( m_status != Job_Failed )
@ -99,7 +90,7 @@ void DownloadJob::downloadFinished()
//TODO: check md5 here! //TODO: check md5 here!
m_reply.clear(); m_reply.clear();
emit finish(); emit succeeded(index_within_job);
return; return;
} }
// else the download failed // else the download failed
@ -111,12 +102,12 @@ void DownloadJob::downloadFinished()
m_output_file.remove(); m_output_file.remove();
} }
m_reply.clear(); m_reply.clear();
emit fail(); emit failed(index_within_job);
return; return;
} }
} }
void DownloadJob::downloadReadyRead() void Download::downloadReadyRead()
{ {
if( m_save_to_file ) if( m_save_to_file )
{ {
@ -128,7 +119,7 @@ void DownloadJob::downloadReadyRead()
* Can't open the file... the job failed * Can't open the file... the job failed
*/ */
m_reply->abort(); m_reply->abort();
emit fail(); emit failed(index_within_job);
return; return;
} }
m_opened_for_saving = true; m_opened_for_saving = true;
@ -136,3 +127,58 @@ void DownloadJob::downloadReadyRead()
m_output_file.write ( m_reply->readAll() ); m_output_file.write ( m_reply->readAll() );
} }
} }
DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 )
{
DownloadPtr ptr (new Download(url, rel_target_path, expected_md5));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
return ptr;
}
void DownloadJob::partSucceeded ( int index )
{
num_succeeded++;
if(num_failed + num_succeeded == downloads.size())
{
if(num_failed)
{
emit failed();
}
else
{
emit succeeded();
}
}
}
void DownloadJob::partFailed ( int index )
{
num_failed++;
if(num_failed + num_succeeded == downloads.size())
{
if(num_failed)
{
emit failed();
}
else
{
emit succeeded();
}
}
}
void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTotal )
{
// PROGRESS? DENIED!
}
void DownloadJob::start()
{
for(auto iter: downloads)
{
connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
iter->start();
}
}

View File

@ -1,28 +1,37 @@
#pragma once #pragma once
#include "JobQueue.h"
#include <QtNetwork> #include <QtNetwork>
/** class DownloadJob;
* A single file for the downloader/cache to process. class Download;
*/ typedef QSharedPointer<DownloadJob> DownloadJobPtr;
class DownloadJob : public Job typedef QSharedPointer<Download> DownloadPtr;
enum JobStatus
{
Job_NotStarted,
Job_InProgress,
Job_Finished,
Job_Failed
};
class Job : public QObject
{ {
Q_OBJECT Q_OBJECT
protected:
explicit Job(): QObject(0){};
public: public:
DownloadJob(QUrl url, virtual ~Job() {};
QString rel_target_path = QString(),
QString expected_md5 = QString()
);
static JobPtr create(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
public slots: public slots:
virtual void start(); virtual void start() = 0;
};
private slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);; class Download: public Job
void downloadError(QNetworkReply::NetworkError error); {
void downloadFinished(); friend class DownloadJob;
void downloadReadyRead(); Q_OBJECT
protected:
Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
public: public:
/// the network reply /// the network reply
QSharedPointer<QNetworkReply> m_reply; QSharedPointer<QNetworkReply> m_reply;
@ -46,6 +55,72 @@ public:
/// if not saving to file, downloaded data is placed here /// if not saving to file, downloaded data is placed here
QByteArray m_data; QByteArray m_data;
int currentProgress = 0;
int totalProgress = 0;
/// The file's status /// The file's status
JobStatus m_status; JobStatus m_status;
int index_within_job = 0;
signals:
void started(int index);
void progress(int index, qint64 current, qint64 total);
void succeeded(int index);
void failed(int index);
public slots:
virtual void start();
private slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
void downloadError(QNetworkReply::NetworkError error);
void downloadFinished();
void downloadReadyRead();
}; };
/**
* A single file for the downloader/cache to process.
*/
class DownloadJob : public Job
{
Q_OBJECT
public:
explicit DownloadJob()
:Job(){};
explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
:Job()
{
add(url, rel_target_path, expected_md5);
};
DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
DownloadPtr operator[](int index)
{
return downloads[index];
};
DownloadPtr first()
{
if(downloads.size())
return downloads[0];
return DownloadPtr();
}
int size() const
{
return downloads.size();
}
signals:
void started();
void progress(qint64 current, qint64 total);
void succeeded();
void failed();
public slots:
virtual void start();
private slots:
void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);;
void partSucceeded(int index);
void partFailed(int index);
private:
QList<DownloadPtr> downloads;
int num_succeeded = 0;
int num_failed = 0;
};

View File

@ -1,180 +0,0 @@
#pragma once
#include <QtCore>
#include "libutil_config.h"
enum JobStatus
{
Job_NotStarted,
Job_InProgress,
Job_Finished,
Job_Failed
};
class JobList;
class Job : public QObject
{
Q_OBJECT
protected:
explicit Job(): QObject(0){};
public:
virtual ~Job() {};
signals:
void finish();
void fail();
void progress(qint64 current, qint64 total);
public slots:
virtual void start() = 0;
};
typedef QSharedPointer<Job> JobPtr;
/**
* A list of jobs, to be processed one by one.
*/
class JobList : public QObject
{
friend class JobListQueue;
Q_OBJECT
public:
JobList() : QObject(0)
{
m_status = Job_NotStarted;
current_job_idx = 0;
}
JobStatus getStatus()
{
return m_status;
}
void add(JobPtr dlable)
{
if(m_status == Job_NotStarted)
m_jobs.append(dlable);
//else there's a bug. TODO: catch the bugs
}
JobPtr getFirstJob()
{
if(m_jobs.size())
return m_jobs[0];
else
return JobPtr();
}
void start()
{
current_job_idx = 0;
auto job = m_jobs[current_job_idx];
connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
job->start();
emit started();
}
private slots:
void currentJobFinished()
{
if(current_job_idx == m_jobs.size() - 1)
{
m_status = Job_Finished;
emit finished();
}
else
{
current_job_idx++;
auto job = m_jobs[current_job_idx];
connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
job->start();
}
}
void currentJobFailed()
{
m_status = Job_Failed;
emit failed();
}
void currentJobProgress(qint64 current, qint64 total)
{
if(!total)
return;
int total_jobs = m_jobs.size();
if(!total_jobs)
return;
float job_chunk = 1000.0 / float(total_jobs);
float cur = current;
float tot = total;
float last_chunk = (cur / tot) * job_chunk;
float list_total = job_chunk * current_job_idx + last_chunk;
emit progress(qint64(list_total), 1000LL);
}
private:
QVector<JobPtr> m_jobs;
/// The overall status of this job list
JobStatus m_status;
int current_job_idx;
signals:
void progress(qint64 current, qint64 total);
void started();
void finished();
void failed();
};
typedef QSharedPointer<JobList> JobListPtr;
/**
* A queue of job lists! The job lists fail or finish as units.
*/
class JobListQueue : public QObject
{
Q_OBJECT
public:
JobListQueue(QObject *p = 0):
QObject(p),
currentIndex(0),
is_running(false){}
void enqueue(JobListPtr job)
{
jobs.enqueue(job);
// finish or fail, we should catch that and start the next one
connect(job.data(),SIGNAL(finished()), SLOT(startNextJob()));
connect(job.data(),SIGNAL(failed()), SLOT(startNextJob()));
if(!is_running)
{
QTimer::singleShot(0, this, SLOT(startNextJob()));
}
}
private slots:
void startNextJob()
{
if (jobs.isEmpty())
{
currentJobList.clear();
currentIndex = 0;
is_running = false;
emit finishedAllJobs();
return;
}
currentJobList = jobs.dequeue();
is_running = true;
currentIndex = 0;
currentJobList->start();
}
signals:
void finishedAllJobs();
private:
JobListPtr currentJobList;
QQueue<JobListPtr> jobs;
unsigned currentIndex;
bool is_running;
};

View File

@ -1,7 +1,25 @@
#include "NetWorker.h" #include "NetWorker.h"
#include <QThreadStorage> #include <QThreadStorage>
NetWorker& NetWorker::spawn() class NetWorker::Private
{
public:
QNetworkAccessManager manager;
};
NetWorker::NetWorker ( QObject* parent ) : QObject ( parent )
{
d = new Private();
}
QNetworkAccessManager& NetWorker::qnam()
{
auto & w = worker();
return w.d->manager;
}
NetWorker& NetWorker::worker()
{ {
static QThreadStorage<NetWorker *> storage; static QThreadStorage<NetWorker *> storage;
if (!storage.hasLocalData()) if (!storage.hasLocalData())

View File

@ -9,12 +9,23 @@
'OOOOOY _.' 'OOOOOY _.'
'""""'' '""""''
*/ */
#pragma once #pragma once
#include <QNetworkAccessManager> #include <QNetworkAccessManager>
class NetWorker : public QNetworkAccessManager #include <QUrl>
class NetWorker : public QObject
{ {
Q_OBJECT Q_OBJECT
public: public:
static NetWorker &spawn(); // for high level access to the sevices (preferred)
static NetWorker &worker();
// for low-level access to the network manager object
static QNetworkAccessManager &qnam();
public:
private:
explicit NetWorker ( QObject* parent = 0 );
class Private;
Private * d;
}; };

View File

@ -29,7 +29,7 @@ LoginTask::LoginTask( const UserInfo& uInfo, QObject* parent ) : Task(parent), u
void LoginTask::executeTask() void LoginTask::executeTask()
{ {
setStatus("Logging in..."); setStatus("Logging in...");
auto & worker = NetWorker::spawn(); auto & worker = NetWorker::qnam();
connect(&worker, SIGNAL(finished(QNetworkReply*)), this, SLOT(processNetReply(QNetworkReply*))); connect(&worker, SIGNAL(finished(QNetworkReply*)), this, SLOT(processNetReply(QNetworkReply*)));
QUrl loginURL("https://login.minecraft.net/"); QUrl loginURL("https://login.minecraft.net/");