From ec87a8ddfc2e6c23ab4c1003f6cafa685ce233d8 Mon Sep 17 00:00:00 2001 From: flow Date: Sat, 16 Jul 2022 20:29:03 -0300 Subject: [PATCH 1/3] fix: add expiration time to cache entries This is to prevent problems where the cache entry would still be used way after the remote resource got updated. The limit is hardcoded for 1 week, which I think is a reasonable time, but this could be further tweaked. Signed-off-by: flow --- launcher/net/HttpMetaCache.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/launcher/net/HttpMetaCache.cpp b/launcher/net/HttpMetaCache.cpp index 4d86c0b8..769f162b 100644 --- a/launcher/net/HttpMetaCache.cpp +++ b/launcher/net/HttpMetaCache.cpp @@ -44,6 +44,11 @@ #include +/** Maximum time to hold a cache entry + * = 1 week in milliseconds + */ +#define TIME_TO_EXPIRE 1*7*24*60*60*1000 + auto MetaEntry::getFullPath() -> QString { // FIXME: make local? @@ -121,6 +126,15 @@ auto HttpMetaCache::resolveEntry(QString base, QString resource_path, QString ex SaveEventually(); } + // Get rid of old entries, to prevent cache problems + auto current_time = QDateTime::currentMSecsSinceEpoch(); + auto remote_time = QDateTime::fromString(entry->remote_changed_timestamp).toMSecsSinceEpoch(); + if (current_time - remote_time < TIME_TO_EXPIRE) { + qWarning() << "Removing cache entry because of old age!"; + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + // entry passed all the checks we cared about. entry->basePath = getBasePath(base); return entry; @@ -240,6 +254,8 @@ void HttpMetaCache::SaveNow() if (m_index_file.isNull()) return; + qDebug() << "[HttpMetaCache]" << "Saving metacache with" << m_entries.size() << "entries"; + QJsonObject toplevel; Json::writeString(toplevel, "version", "1"); From c8a72c876da69de669d5616dd7ff755f157baf99 Mon Sep 17 00:00:00 2001 From: flow Date: Sat, 16 Jul 2022 20:36:14 -0300 Subject: [PATCH 2/3] fix: add missing HttpMetaCache entry for CF mods Signed-off-by: flow --- launcher/Application.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/launcher/Application.cpp b/launcher/Application.cpp index 2bd91fd7..41be6a5b 100644 --- a/launcher/Application.cpp +++ b/launcher/Application.cpp @@ -864,6 +864,7 @@ Application::Application(int &argc, char **argv) : QApplication(argc, argv) m_metacache->addBase("ModpacksCHPacks", QDir("cache/ModpacksCHPacks").absolutePath()); m_metacache->addBase("TechnicPacks", QDir("cache/TechnicPacks").absolutePath()); m_metacache->addBase("FlamePacks", QDir("cache/FlamePacks").absolutePath()); + m_metacache->addBase("FlameMods", QDir("cache/FlameMods").absolutePath()); m_metacache->addBase("ModrinthPacks", QDir("cache/ModrinthPacks").absolutePath()); m_metacache->addBase("root", QDir::currentPath()); m_metacache->addBase("translations", QDir("translations").absolutePath()); From ab6e1b112b05a1fcad962d15c087ed9ae746bf87 Mon Sep 17 00:00:00 2001 From: flow Date: Sun, 17 Jul 2022 11:24:12 -0300 Subject: [PATCH 3/3] change(cache): use cache-specific http headers for their lifetime This uses the 'Age', 'Cache-Control' and 'Expires' HTTP headers to more accurately set up the cache lifetime, falling back to a static 1-week time if they're not present in the response. Signed-off-by: flow --- launcher/net/HttpMetaCache.cpp | 14 ++++++------- launcher/net/HttpMetaCache.h | 10 +++++++++ launcher/net/MetaCacheSink.cpp | 38 +++++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/launcher/net/HttpMetaCache.cpp b/launcher/net/HttpMetaCache.cpp index 769f162b..deb2780b 100644 --- a/launcher/net/HttpMetaCache.cpp +++ b/launcher/net/HttpMetaCache.cpp @@ -44,11 +44,6 @@ #include -/** Maximum time to hold a cache entry - * = 1 week in milliseconds - */ -#define TIME_TO_EXPIRE 1*7*24*60*60*1000 - auto MetaEntry::getFullPath() -> QString { // FIXME: make local? @@ -127,9 +122,8 @@ auto HttpMetaCache::resolveEntry(QString base, QString resource_path, QString ex } // Get rid of old entries, to prevent cache problems - auto current_time = QDateTime::currentMSecsSinceEpoch(); - auto remote_time = QDateTime::fromString(entry->remote_changed_timestamp).toMSecsSinceEpoch(); - if (current_time - remote_time < TIME_TO_EXPIRE) { + auto current_time = QDateTime::currentSecsSinceEpoch(); + if (entry->isExpired(current_time - ( file_last_changed / 1000 ))) { qWarning() << "Removing cache entry because of old age!"; selected_base.entry_list.remove(resource_path); return staleEntry(base, resource_path); @@ -235,6 +229,8 @@ void HttpMetaCache::Load() foo->etag = Json::ensureString(element_obj, "etag"); foo->local_changed_timestamp = Json::ensureDouble(element_obj, "last_changed_timestamp"); foo->remote_changed_timestamp = Json::ensureString(element_obj, "remote_changed_timestamp"); + foo->current_age = Json::ensureDouble(element_obj, "current_age"); + foo->max_age = Json::ensureDouble(element_obj, "max_age"); // presumed innocent until closer examination foo->stale = false; @@ -275,6 +271,8 @@ void HttpMetaCache::SaveNow() entryObj.insert("last_changed_timestamp", QJsonValue(double(entry->local_changed_timestamp))); if (!entry->remote_changed_timestamp.isEmpty()) entryObj.insert("remote_changed_timestamp", QJsonValue(entry->remote_changed_timestamp)); + entryObj.insert("current_age", QJsonValue(double(entry->current_age))); + entryObj.insert("max_age", QJsonValue(double(entry->max_age))); entriesArr.append(entryObj); } } diff --git a/launcher/net/HttpMetaCache.h b/launcher/net/HttpMetaCache.h index e944b3d5..df3549e8 100644 --- a/launcher/net/HttpMetaCache.h +++ b/launcher/net/HttpMetaCache.h @@ -64,6 +64,14 @@ class MetaEntry { auto getMD5Sum() -> QString { return md5sum; } void setMD5Sum(QString md5sum) { this->md5sum = md5sum; } + auto getCurrentAge() -> qint64 { return current_age; } + void setCurrentAge(qint64 age) { current_age = age; } + + auto getMaximumAge() -> qint64 { return max_age; } + void setMaximumAge(qint64 age) { max_age = age; } + + bool isExpired(qint64 offset) { return current_age >= max_age - offset; }; + protected: QString baseId; QString basePath; @@ -72,6 +80,8 @@ class MetaEntry { QString etag; qint64 local_changed_timestamp = 0; QString remote_changed_timestamp; // QString for now, RFC 2822 encoded time + qint64 current_age = 0; + qint64 max_age = 0; bool stale = true; }; diff --git a/launcher/net/MetaCacheSink.cpp b/launcher/net/MetaCacheSink.cpp index f86dd870..ab0c9fcb 100644 --- a/launcher/net/MetaCacheSink.cpp +++ b/launcher/net/MetaCacheSink.cpp @@ -36,11 +36,16 @@ #include "MetaCacheSink.h" #include #include -#include "FileSystem.h" #include "Application.h" namespace Net { +/** Maximum time to hold a cache entry + * = 1 week in seconds + */ +#define MAX_TIME_TO_EXPIRE 1*7*24*60*60 + + MetaCacheSink::MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum) :Net::FileSink(entry->getFullPath()), m_entry(entry), m_md5Node(md5sum) { @@ -88,6 +93,37 @@ Task::State MetaCacheSink::finalizeCache(QNetworkReply & reply) } m_entry->setLocalChangedTimestamp(output_file_info.lastModified().toUTC().toMSecsSinceEpoch()); + + { // Cache lifetime + if (reply.hasRawHeader("Cache-Control")) { + auto cache_control_header = reply.rawHeader("Cache-Control"); + // qDebug() << "[MetaCache] Parsing 'Cache-Control' header with" << cache_control_header; + + QRegularExpression max_age_expr("max-age=([0-9]+)"); + qint64 max_age = max_age_expr.match(cache_control_header).captured(1).toLongLong(); + m_entry->setMaximumAge(max_age); + + } else if (reply.hasRawHeader("Expires")) { + auto expires_header = reply.rawHeader("Expires"); + // qDebug() << "[MetaCache] Parsing 'Expires' header with" << expires_header; + + qint64 max_age = QDateTime::fromString(expires_header).toSecsSinceEpoch() - QDateTime::currentSecsSinceEpoch(); + m_entry->setMaximumAge(max_age); + } else { + m_entry->setMaximumAge(MAX_TIME_TO_EXPIRE); + } + + if (reply.hasRawHeader("Age")) { + auto age_header = reply.rawHeader("Age"); + // qDebug() << "[MetaCache] Parsing 'Age' header with" << age_header; + + qint64 current_age = age_header.toLongLong(); + m_entry->setCurrentAge(current_age); + } else { + m_entry->setCurrentAge(0); + } + } + m_entry->setStale(false); APPLICATION->metacache()->updateEntry(m_entry);