0xf8/pollymc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use HttpMetaCache to minimize network use.

Browse Source
This commit is contained in:
Petr Mrázek
2013-09-08 02:15:20 +02:00
parent 6892c11e9f
commit 6bea4ec988
17 changed files with 698 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
logic/net/ByteArrayDownload.cpp Normal file
View File

@@ -0,0 +1,62 @@
#include "ByteArrayDownload.h"
#include "MultiMC.h"
#include <QDebug>
ByteArrayDownload::ByteArrayDownload ( QUrl url )
: Download()
{
m_url = url;
m_status = Job_NotStarted;
}
void ByteArrayDownload::start()
{
qDebug() << "Downloading " << m_url.toString();
QNetworkRequest request ( m_url );
auto worker = MMC->qnam();
QNetworkReply * rep = worker->get ( request );
m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
}
void ByteArrayDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
{
emit progress (index_within_job, bytesReceived, bytesTotal );
}
void ByteArrayDownload::downloadError ( QNetworkReply::NetworkError error )
{
// error happened during download.
// TODO: log the reason why
m_status = Job_Failed;
}
void ByteArrayDownload::downloadFinished()
{
// if the download succeeded
if ( m_status != Job_Failed )
{
// nothing went wrong...
m_status = Job_Finished;
m_data = m_reply->readAll();
m_reply.clear();
emit succeeded(index_within_job);
return;
}
// else the download failed
else
{
m_reply.clear();
emit failed(index_within_job);
return;
}
}
void ByteArrayDownload::downloadReadyRead()
{
// ~_~
}

24
logic/net/ByteArrayDownload.h Normal file
View File

@@ -0,0 +1,24 @@
#pragma once
#include "Download.h"
class ByteArrayDownload: public Download
{
Q_OBJECT
public:
ByteArrayDownload(QUrl url);
public:
/// if not saving to file, downloaded data is placed here
QByteArray m_data;
public slots:
virtual void start();
protected slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
void downloadError(QNetworkReply::NetworkError error);
void downloadFinished();
void downloadReadyRead();
};
typedef QSharedPointer<ByteArrayDownload> ByteArrayDownloadPtr;

122
logic/net/CacheDownload.cpp Normal file
View File

@@ -0,0 +1,122 @@
#include "MultiMC.h"
#include "CacheDownload.h"
#include <pathutils.h>
#include <QCryptographicHash>
#include <QFileInfo>
#include <QDateTime>
#include <QDebug>
CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry )
:Download(), md5sum(QCryptographicHash::Md5)
{
m_url = url;
m_entry = entry;
m_target_path = entry->getFullPath();
m_status = Job_NotStarted;
m_opened_for_saving = false;
}
void CacheDownload::start()
{
if(!m_entry->stale)
{
emit succeeded(index_within_job);
return;
}
m_output_file.setFileName(m_target_path);
// if there already is a file and md5 checking is in effect and it can be opened
if(!ensureFilePathExists(m_target_path))
{
emit failed(index_within_job);
return;
}
qDebug() << "Downloading " << m_url.toString();
QNetworkRequest request ( m_url );
request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1());
auto worker = MMC->qnam();
QNetworkReply * rep = worker->get ( request );
m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
}
void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
{
emit progress (index_within_job, bytesReceived, bytesTotal );
}
void CacheDownload::downloadError ( QNetworkReply::NetworkError error )
{
// error happened during download.
// TODO: log the reason why
m_status = Job_Failed;
}
void CacheDownload::downloadFinished()
{
// if the download succeeded
if ( m_status != Job_Failed )
{
// nothing went wrong...
m_status = Job_Finished;
if(m_opened_for_saving)
{
// save the data to the downloadable if we aren't saving to file
m_output_file.close();
m_entry->md5sum = md5sum.result().toHex().constData();
}
else
{
if ( m_output_file.open ( QIODevice::ReadOnly ) )
{
m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
m_output_file.close();
}
}
QFileInfo output_file_info(m_target_path);
m_entry->etag = m_reply->rawHeader("ETag").constData();
m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch();
m_entry->stale = false;
MMC->metacache()->updateEntry(m_entry);
m_reply.clear();
emit succeeded(index_within_job);
return;
}
// else the download failed
else
{
m_output_file.close();
m_output_file.remove();
m_reply.clear();
emit failed(index_within_job);
return;
}
}
void CacheDownload::downloadReadyRead()
{
if(!m_opened_for_saving)
{
if ( !m_output_file.open ( QIODevice::WriteOnly ) )
{
/*
* Can't open the file... the job failed
*/
m_reply->abort();
emit failed(index_within_job);
return;
}
m_opened_for_saving = true;
}
QByteArray ba = m_reply->readAll();
md5sum.addData(ba);
m_output_file.write ( ba );
}

34
logic/net/CacheDownload.h Normal file
View File

@@ -0,0 +1,34 @@
#pragma once
#include "Download.h"
#include "HttpMetaCache.h"
#include <QFile>
#include <qcryptographichash.h>
class CacheDownload : public Download
{
Q_OBJECT
public:
MetaEntryPtr m_entry;
/// is the saving file already open?
bool m_opened_for_saving;
/// if saving to file, use the one specified in this string
QString m_target_path;
/// this is the output file, if any
QFile m_output_file;
/// the hash-as-you-download
QCryptographicHash md5sum;
public:
explicit CacheDownload(QUrl url, MetaEntryPtr entry);
protected slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished();
virtual void downloadReadyRead();
public slots:
virtual void start();
};
typedef QSharedPointer<CacheDownload> CacheDownloadPtr;

54
logic/net/Download.h Normal file
View File

@@ -0,0 +1,54 @@
#pragma once
#include <QObject>
#include <QUrl>
#include <QSharedPointer>
#include <QNetworkReply>
enum JobStatus
{
Job_NotStarted,
Job_InProgress,
Job_Finished,
Job_Failed
};
class Download : public QObject
{
Q_OBJECT
protected:
explicit Download(): QObject(0){};
public:
virtual ~Download() {};
public:
/// the network reply
QSharedPointer<QNetworkReply> m_reply;
/// source URL
QUrl m_url;
/// The file's status
JobStatus m_status;
/// index within the parent job
int index_within_job = 0;
signals:
void started(int index);
void progress(int index, qint64 current, qint64 total);
void succeeded(int index);
void failed(int index);
protected slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0;
virtual void downloadError(QNetworkReply::NetworkError error) = 0;
virtual void downloadFinished() = 0;
virtual void downloadReadyRead() = 0;
public slots:
virtual void start() = 0;
};
typedef QSharedPointer<Download> DownloadPtr;

146
logic/net/DownloadJob.cpp
View File

@@ -1,136 +1,29 @@
#include "DownloadJob.h"
#include "pathutils.h"
#include "MultiMC.h"
#include "FileDownload.h"
#include "ByteArrayDownload.h"
#include "CacheDownload.h"
Download::Download (QUrl url, QString target_path, QString expected_md5 )
:Job()
ByteArrayDownloadPtr DownloadJob::add ( QUrl url )
{
m_url = url;
m_target_path = target_path;
m_expected_md5 = expected_md5;
m_check_md5 = m_expected_md5.size();
m_save_to_file = m_target_path.size();
m_status = Job_NotStarted;
m_opened_for_saving = false;
ByteArrayDownloadPtr ptr (new ByteArrayDownload(url));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
return ptr;
}
void Download::start()
FileDownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path)
{
if ( m_save_to_file )
{
QString filename = m_target_path;
m_output_file.setFileName ( filename );
// if there already is a file and md5 checking is in effect and it can be opened
if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) )
{
// check the md5 against the expected one
QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
m_output_file.close();
// skip this file if they match
if ( m_check_md5 && hash == m_expected_md5 )
{
qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
emit succeeded(index_within_job);
return;
}
else
{
m_expected_md5 = hash;
}
}
if(!ensureFilePathExists(filename))
{
emit failed(index_within_job);
return;
}
}
qDebug() << "Downloading " << m_url.toString();
QNetworkRequest request ( m_url );
request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1());
auto worker = MMC->qnam();
QNetworkReply * rep = worker->get ( request );
m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
FileDownloadPtr ptr (new FileDownload(url, rel_target_path));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
return ptr;
}
void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
CacheDownloadPtr DownloadJob::add ( QUrl url, MetaEntryPtr entry)
{
emit progress (index_within_job, bytesReceived, bytesTotal );
}
void Download::downloadError ( QNetworkReply::NetworkError error )
{
// error happened during download.
// TODO: log the reason why
m_status = Job_Failed;
}
void Download::downloadFinished()
{
// if the download succeeded
if ( m_status != Job_Failed )
{
// nothing went wrong...
m_status = Job_Finished;
// save the data to the downloadable if we aren't saving to file
if ( !m_save_to_file )
{
m_data = m_reply->readAll();
}
else
{
m_output_file.close();
}
//TODO: check md5 here!
m_reply.clear();
emit succeeded(index_within_job);
return;
}
// else the download failed
else
{
if ( m_save_to_file )
{
m_output_file.close();
m_output_file.remove();
}
m_reply.clear();
emit failed(index_within_job);
return;
}
}
void Download::downloadReadyRead()
{
if( m_save_to_file )
{
if(!m_opened_for_saving)
{
if ( !m_output_file.open ( QIODevice::WriteOnly ) )
{
/*
* Can't open the file... the job failed
*/
m_reply->abort();
emit failed(index_within_job);
return;
}
m_opened_for_saving = true;
}
m_output_file.write ( m_reply->readAll() );
}
}
DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 )
{
DownloadPtr ptr (new Download(url, rel_target_path, expected_md5));
CacheDownloadPtr ptr (new CacheDownload(url, entry));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
return ptr;
@@ -139,16 +32,17 @@ DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expect
void DownloadJob::partSucceeded ( int index )
{
num_succeeded++;
qDebug() << m_job_name.toLocal8Bit() << " progress: " << num_succeeded << "/" << downloads.size();
if(num_failed + num_succeeded == downloads.size())
{
if(num_failed)
{
qDebug() << "Download JOB failed: " << this;
qDebug() << m_job_name.toLocal8Bit() << " failed.";
emit failed();
}
else
{
qDebug() << "Download JOB succeeded: " << this;
qDebug() << m_job_name.toLocal8Bit() << " succeeded.";
emit succeeded();
}
}
@@ -159,7 +53,7 @@ void DownloadJob::partFailed ( int index )
num_failed++;
if(num_failed + num_succeeded == downloads.size())
{
qDebug() << "Download JOB failed: " << this;
qDebug() << m_job_name.toLocal8Bit() << " failed.";
emit failed();
}
}
@@ -172,7 +66,7 @@ void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTo
void DownloadJob::start()
{
qDebug() << "Download JOB started: " << this;
qDebug() << m_job_name.toLocal8Bit() << " started.";
for(auto iter: downloads)
{
connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));

95
logic/net/DownloadJob.h
View File

@@ -1,98 +1,28 @@
#pragma once
#include <QtNetwork>
#include "Download.h"
#include "ByteArrayDownload.h"
#include "FileDownload.h"
#include "CacheDownload.h"
#include "HttpMetaCache.h"
class DownloadJob;
class Download;
typedef QSharedPointer<DownloadJob> DownloadJobPtr;
typedef QSharedPointer<Download> DownloadPtr;
enum JobStatus
{
Job_NotStarted,
Job_InProgress,
Job_Finished,
Job_Failed
};
class Job : public QObject
{
Q_OBJECT
protected:
explicit Job(): QObject(0){};
public:
virtual ~Job() {};
public slots:
virtual void start() = 0;
};
class Download: public Job
{
friend class DownloadJob;
Q_OBJECT
protected:
Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
public:
/// the network reply
QSharedPointer<QNetworkReply> m_reply;
/// source URL
QUrl m_url;
/// if true, check the md5sum against a provided md5sum
/// also, if a file exists, perform an md5sum first and don't download only if they don't match
bool m_check_md5;
/// the expected md5 checksum
QString m_expected_md5;
/// save to file?
bool m_save_to_file;
/// is the saving file already open?
bool m_opened_for_saving;
/// if saving to file, use the one specified in this string
QString m_target_path;
/// this is the output file, if any
QFile m_output_file;
/// if not saving to file, downloaded data is placed here
QByteArray m_data;
int currentProgress = 0;
int totalProgress = 0;
/// The file's status
JobStatus m_status;
int index_within_job = 0;
signals:
void started(int index);
void progress(int index, qint64 current, qint64 total);
void succeeded(int index);
void failed(int index);
public slots:
virtual void start();
private slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
void downloadError(QNetworkReply::NetworkError error);
void downloadFinished();
void downloadReadyRead();
};
/**
* A single file for the downloader/cache to process.
*/
class DownloadJob : public Job
class DownloadJob : public QObject
{
Q_OBJECT
public:
explicit DownloadJob()
:Job(){};
explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
:Job()
{
add(url, rel_target_path, expected_md5);
};
explicit DownloadJob(QString job_name)
:QObject(), m_job_name(job_name){};
ByteArrayDownloadPtr add(QUrl url);
FileDownloadPtr add(QUrl url, QString rel_target_path);
CacheDownloadPtr add(QUrl url, MetaEntryPtr entry);
DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
DownloadPtr operator[](int index)
{
return downloads[index];
@@ -119,6 +49,7 @@ private slots:
void partSucceeded(int index);
void partFailed(int index);
private:
QString m_job_name;
QList<DownloadPtr> downloads;
int num_succeeded = 0;
int num_failed = 0;

111
logic/net/FileDownload.cpp Normal file
View File

@@ -0,0 +1,111 @@
#include "MultiMC.h"
#include "FileDownload.h"
#include <pathutils.h>
#include <QCryptographicHash>
#include <QDebug>
FileDownload::FileDownload ( QUrl url, QString target_path )
:Download()
{
m_url = url;
m_target_path = target_path;
m_check_md5 = false;
m_status = Job_NotStarted;
m_opened_for_saving = false;
}
void FileDownload::start()
{
QString filename = m_target_path;
m_output_file.setFileName ( filename );
// if there already is a file and md5 checking is in effect and it can be opened
if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) )
{
// check the md5 against the expected one
QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
m_output_file.close();
// skip this file if they match
if ( m_check_md5 && hash == m_expected_md5 )
{
qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
emit succeeded(index_within_job);
return;
}
else
{
m_expected_md5 = hash;
}
}
if(!ensureFilePathExists(filename))
{
emit failed(index_within_job);
return;
}
qDebug() << "Downloading " << m_url.toString();
QNetworkRequest request ( m_url );
request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1());
auto worker = MMC->qnam();
QNetworkReply * rep = worker->get ( request );
m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
}
void FileDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
{
emit progress (index_within_job, bytesReceived, bytesTotal );
}
void FileDownload::downloadError ( QNetworkReply::NetworkError error )
{
// error happened during download.
// TODO: log the reason why
m_status = Job_Failed;
}
void FileDownload::downloadFinished()
{
// if the download succeeded
if ( m_status != Job_Failed )
{
// nothing went wrong...
m_status = Job_Finished;
m_output_file.close();
m_reply.clear();
emit succeeded(index_within_job);
return;
}
// else the download failed
else
{
m_output_file.close();
m_reply.clear();
emit failed(index_within_job);
return;
}
}
void FileDownload::downloadReadyRead()
{
if(!m_opened_for_saving)
{
if ( !m_output_file.open ( QIODevice::WriteOnly ) )
{
/*
* Can't open the file... the job failed
*/
m_reply->abort();
emit failed(index_within_job);
return;
}
m_opened_for_saving = true;
}
m_output_file.write ( m_reply->readAll() );
}

35
logic/net/FileDownload.h Normal file
View File

@@ -0,0 +1,35 @@
#pragma once
#include "Download.h"
#include <QFile>
class FileDownload : public Download
{
Q_OBJECT
public:
/// if true, check the md5sum against a provided md5sum
/// also, if a file exists, perform an md5sum first and don't download only if they don't match
bool m_check_md5;
/// the expected md5 checksum
QString m_expected_md5;
/// is the saving file already open?
bool m_opened_for_saving;
/// if saving to file, use the one specified in this string
QString m_target_path;
/// this is the output file, if any
QFile m_output_file;
public:
explicit FileDownload(QUrl url, QString target_path);
protected slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished();
virtual void downloadReadyRead();
public slots:
virtual void start();
};
typedef QSharedPointer<FileDownload> FileDownloadPtr;

167
logic/net/HttpMetaCache.cpp
View File

@@ -1,38 +1,136 @@
#include "MultiMC.h"
#include "HttpMetaCache.h"
#include <pathutils.h>
#include <QFileInfo>
#include <QFile>
#include <qjsondocument.h>
#include <qjsonarray.h>
#include <qjsonobject.h>
#include <qfileinfo.h>
#include <qtemporaryfile.h>
#include <qsavefile.h>
#include <QTemporaryFile>
#include <QSaveFile>
#include <QDateTime>
#include <QCryptographicHash>
#include <QDebug>
#include <QJsonDocument>
#include <QJsonArray>
#include <QJsonObject>
QString MetaEntry::getFullPath()
{
return PathCombine(MMC->metacache()->getBasePath(base), path);
}
HttpMetaCache::HttpMetaCache(QString path)
:QObject()
{
m_index_file = path;
}
HttpMetaCache::~HttpMetaCache()
{
Save();
saveBatchingTimer.setSingleShot(true);
saveBatchingTimer.setTimerType(Qt::VeryCoarseTimer);
connect(&saveBatchingTimer,SIGNAL(timeout()),SLOT(SaveNow()));
}
void HttpMetaCache::addEntry ( QString base, QString resource_path, QString etag )
HttpMetaCache::~HttpMetaCache()
{
saveBatchingTimer.stop();
SaveNow();
}
void HttpMetaCache::SaveEventually()
{
saveBatchingTimer.stop();
saveBatchingTimer.start(30000);
}
MetaEntryPtr HttpMetaCache::getEntry ( QString base, QString resource_path )
{
// no base. no base path. can't store
if(!m_entries.contains(base))
return;
QString real_path = PathCombine(m_entries[base].base_path, resource_path);
QFileInfo finfo(real_path);
// just ignore it, it's garbage if it's not a proper file
if(!finfo.isFile() || !finfo.isReadable())
{
// TODO: log problem
return;
return MetaEntryPtr();
}
EntryMap & map = m_entries[base];
if(map.entry_list.contains(resource_path))
{
return map.entry_list[resource_path];
}
return MetaEntryPtr();
}
MetaEntryPtr HttpMetaCache::resolveEntry ( QString base, QString resource_path, QString expected_etag )
{
auto entry = getEntry(base, resource_path);
// it's not present? generate a default stale entry
if(!entry)
{
return staleEntry(base, resource_path);
}
Save();
auto & selected_base = m_entries[base];
QString real_path = PathCombine(selected_base.base_path, resource_path);
QFileInfo finfo(real_path);
// is the file really there? if not -> stale
if(!finfo.isFile() || !finfo.isReadable())
{
// if the file doesn't exist, we disown the entry
selected_base.entry_list.remove(resource_path);
return staleEntry(base, resource_path);
}
if(!expected_etag.isEmpty() && expected_etag != entry->etag)
{
// if the etag doesn't match expected, we disown the entry
selected_base.entry_list.remove(resource_path);
return staleEntry(base, resource_path);
}
// if the file changed, check md5sum
qint64 file_last_changed = finfo.lastModified().toUTC().toMSecsSinceEpoch();
if(file_last_changed != entry->last_changed_timestamp)
{
QFile input(real_path);
input.open(QIODevice::ReadOnly);
QString md5sum = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Md5).toHex().constData();
if(entry->md5sum != md5sum)
{
selected_base.entry_list.remove(resource_path);
return staleEntry(base, resource_path);
}
// md5sums matched... keep entry and save the new state to file
entry->last_changed_timestamp = file_last_changed;
SaveEventually();
}
// entry passed all the checks we cared about.
return entry;
}
bool HttpMetaCache::updateEntry ( MetaEntryPtr stale_entry )
{
if(!m_entries.contains(stale_entry->base))
{
qDebug() << "Cannot add entry with unknown base: " << stale_entry->base.toLocal8Bit();
return false;
}
if(stale_entry->stale)
{
qDebug() << "Cannot add stale entry: " << stale_entry->getFullPath().toLocal8Bit();
return false;
}
m_entries[stale_entry->base].entry_list[stale_entry->path] = stale_entry;
SaveEventually();
return true;
}
MetaEntryPtr HttpMetaCache::staleEntry(QString base, QString resource_path)
{
auto foo = new MetaEntry;
foo->base = base;
foo->path = resource_path;
foo->stale = true;
return MetaEntryPtr(foo);
}
void HttpMetaCache::addBase ( QString base, QString base_root )
@@ -46,6 +144,16 @@ void HttpMetaCache::addBase ( QString base, QString base_root )
m_entries[base] = foo;
}
QString HttpMetaCache::getBasePath ( QString base )
{
if(m_entries.contains(base))
{
return m_entries[base].base_path;
}
return QString();
}
void HttpMetaCache::Load()
{
QFile index(m_index_file);
@@ -65,12 +173,12 @@ void HttpMetaCache::Load()
// read the entry array
auto entries_val =root.value("entries");
if(!version_val.isArray())
if(!entries_val.isArray())
return;
QJsonArray array = json.array();
QJsonArray array = entries_val.toArray();
for(auto element: array)
{
if(!element.isObject());
if(!element.isObject())
return;
auto element_obj = element.toObject();
QString base = element_obj.value("base").toString();
@@ -83,11 +191,13 @@ void HttpMetaCache::Load()
foo->md5sum = element_obj.value("md5sum").toString();
foo->etag = element_obj.value("etag").toString();
foo->last_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble();
// presumed innocent until closer examination
foo->stale = false;
entrymap.entry_list[path] = MetaEntryPtr( foo );
}
}
void HttpMetaCache::Save()
void HttpMetaCache::SaveNow()
{
QSaveFile tfile(m_index_file);
if(!tfile.open(QIODevice::WriteOnly | QIODevice::Truncate))
@@ -118,14 +228,3 @@ void HttpMetaCache::Save()
return;
tfile.commit();
}
MetaEntryPtr HttpMetaCache::getEntryForResource ( QString base, QString resource_path )
{
if(!m_entries.contains(base))
return MetaEntryPtr();
auto & entrymap = m_entries[base];
if(!entrymap.entry_list.contains(resource_path))
return MetaEntryPtr();
return entrymap.entry_list[resource_path];
}

33
logic/net/HttpMetaCache.h
View File

@@ -2,6 +2,7 @@
#include <QString>
#include <QSharedPointer>
#include <QMap>
#include <qtimer.h>
struct MetaEntry
{
@@ -9,23 +10,42 @@ struct MetaEntry
QString path;
QString md5sum;
QString etag;
quint64 last_changed_timestamp = 0;
qint64 last_changed_timestamp = 0;
bool stale = true;
QString getFullPath();
};
typedef QSharedPointer<MetaEntry> MetaEntryPtr;
class HttpMetaCache
class HttpMetaCache : public QObject
{
Q_OBJECT
public:
// supply path to the cache index file
HttpMetaCache(QString path);
~HttpMetaCache();
MetaEntryPtr getEntryForResource(QString base, QString resource_path);
void addEntry(QString base, QString resource_path, QString etag);
// get the entry solely from the cache
// you probably don't want this, unless you have some specific caching needs.
MetaEntryPtr getEntry(QString base, QString resource_path);
// get the entry from cache and verify that it isn't stale (within reason)
MetaEntryPtr resolveEntry(QString base, QString resource_path, QString expected_etag = QString());
// add a previously resolved stale entry
bool updateEntry(MetaEntryPtr stale_entry);
void addBase(QString base, QString base_root);
private:
void Save();
// (re)start a timer that calls SaveNow later.
void SaveEventually();
void Load();
QString getBasePath ( QString base );
public slots:
void SaveNow();
private:
// create a new stale entry, given the parameters
MetaEntryPtr staleEntry(QString base, QString resource_path);
struct EntryMap
{
QString base_path;
@@ -33,4 +53,5 @@ private:
};
QMap<QString, EntryMap> m_entries;
QString m_index_file;
QTimer saveBatchingTimer;
};