NOISSUE basic code for downloading JREs from Mojang

Not integrated yet, but the logic has tests and shouldn't
be too shaky. Integration comes next.
This commit is contained in:
Petr Mrázek 2020-10-11 23:20:35 +02:00
parent 5180536cc3
commit 0946c7c138
7 changed files with 888 additions and 0 deletions

View File

@ -306,6 +306,9 @@ set(MINECRAFT_SOURCES
# Skin upload utilities # Skin upload utilities
minecraft/SkinUpload.cpp minecraft/SkinUpload.cpp
minecraft/SkinUpload.h minecraft/SkinUpload.h
mojang/PackageManifest.h
mojang/PackageManifest.cpp
) )
add_unit_test(GradleSpecifier add_unit_test(GradleSpecifier
@ -313,6 +316,22 @@ add_unit_test(GradleSpecifier
LIBS MultiMC_logic LIBS MultiMC_logic
) )
add_executable(PackageManifest
mojang/PackageManifest_test.cpp
)
target_link_libraries(PackageManifest
MultiMC_logic
Qt5::Test
)
target_include_directories(PackageManifest
PRIVATE ../../cmake/UnitTest/
)
add_test(
NAME PackageManifest
COMMAND PackageManifest
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)
add_unit_test(MojangVersionFormat add_unit_test(MojangVersionFormat
SOURCES minecraft/MojangVersionFormat_test.cpp SOURCES minecraft/MojangVersionFormat_test.cpp
LIBS MultiMC_logic LIBS MultiMC_logic

View File

@ -0,0 +1,366 @@
#include "PackageManifest.h"
#include <Json.h>
#include <QDir>
#include <QDirIterator>
#include <QCryptographicHash>
#include <QDebug>
namespace mojang_files {
const Hash hash_of_empty_string = "da39a3ee5e6b4b0d3255bfef95601890afd80709";
int Path::compare(const Path& rhs) const
{
auto left_cursor = begin();
auto left_end = end();
auto right_cursor = rhs.begin();
auto right_end = rhs.end();
while (left_cursor != left_end && right_cursor != right_end)
{
if(*left_cursor < *right_cursor)
{
return -1;
}
else if(*left_cursor > *right_cursor)
{
return 1;
}
left_cursor++;
right_cursor++;
}
if(left_cursor == left_end)
{
if(right_cursor == right_end)
{
return 0;
}
return -1;
}
return 1;
}
void Package::addFile(const Path& path, const File& file) {
addFolder(path.parent_path());
files[path] = file;
}
void Package::addFolder(Path folder) {
if(!folder.has_parent_path()) {
return;
}
do {
folders.insert(folder);
folder = folder.parent_path();
} while(folder.has_parent_path());
}
void Package::addLink(const Path& path, const Path& target) {
addFolder(path.parent_path());
symlinks[path] = target;
}
void Package::addSource(const FileSource& source) {
sources[source.hash] = source;
}
namespace {
void fromJson(QJsonDocument & doc, Package & out) {
std::set<Path> seen_paths;
if (!doc.isObject())
{
throw JSONValidationError("file manifest is not an object");
}
QJsonObject root = doc.object();
auto filesObj = Json::ensureObject(root, "files");
auto iter = filesObj.begin();
while (iter != filesObj.end())
{
Path objectPath = Path(iter.key());
auto value = iter.value();
iter++;
if(seen_paths.count(objectPath)) {
throw JSONValidationError("duplicate path inside manifest, the manifest is invalid");
}
if (!value.isObject())
{
throw JSONValidationError("file entry inside manifest is not an an object");
}
seen_paths.insert(objectPath);
auto fileObject = value.toObject();
auto type = Json::requireString(fileObject, "type");
if(type == "directory") {
out.addFolder(objectPath);
continue;
}
else if(type == "file") {
FileSource bestSource;
File file;
file.executable = Json::ensureBoolean(fileObject, "executable", false);
auto downloads = Json::requireObject(fileObject, "downloads");
for(auto iter2 = downloads.begin(); iter2 != downloads.end(); iter2++) {
FileSource source;
auto downloadObject = Json::requireObject(iter2.value());
source.hash = Json::requireString(downloadObject, "sha1");
source.size = Json::requireInteger(downloadObject, "size");
source.url = Json::requireString(downloadObject, "url");
auto compression = iter2.key();
if(compression == "raw") {
file.hash = source.hash;
file.size = source.size;
source.compression = Compression::Raw;
}
else if (compression == "lzma") {
source.compression = Compression::Lzma;
}
else {
continue;
}
bestSource.upgrade(source);
}
if(bestSource.isBad()) {
throw JSONValidationError("No valid compression method for file " + iter.key());
}
out.addFile(objectPath, file);
out.addSource(bestSource);
}
else if(type == "link") {
auto target = Json::requireString(fileObject, "target");
out.symlinks[objectPath] = target;
out.addLink(objectPath, target);
}
else {
throw JSONValidationError("Invalid item type in manifest: " + type);
}
}
// make sure the containing folder exists
out.folders.insert(Path());
}
}
Package Package::fromManifestContents(const QByteArray& contents)
{
Package out;
try
{
auto doc = Json::requireDocument(contents, "Manifest");
fromJson(doc, out);
return out;
}
catch (const Exception &e)
{
qDebug() << QString("Unable to parse manifest: %1").arg(e.cause());
out.valid = false;
return out;
}
}
Package Package::fromManifestFile(const QString & filename) {
Package out;
try
{
auto doc = Json::requireDocument(filename, filename);
fromJson(doc, out);
return out;
}
catch (const Exception &e)
{
qDebug() << QString("Unable to parse manifest file %1: %2").arg(filename, e.cause());
out.valid = false;
return out;
}
}
// FIXME: Qt filesystem abstraction is bad, but ... let's hope it doesn't break too much?
// FIXME: The error handling is just DEFICIENT
Package Package::fromInspectedFolder(const QString& folderPath)
{
QDir root(folderPath);
Package out;
QDirIterator iterator(folderPath, QDir::NoDotAndDotDot | QDir::AllEntries | QDir::System | QDir::Hidden, QDirIterator::Subdirectories);
while(iterator.hasNext()) {
iterator.next();
auto fileInfo = iterator.fileInfo();
auto relPath = root.relativeFilePath(fileInfo.filePath());
if(fileInfo.isSymLink()) {
out.addLink(relPath, fileInfo.symLinkTarget());
}
else if(fileInfo.isDir()) {
out.addFolder(relPath);
}
else if(fileInfo.isFile()) {
File f;
f.executable = fileInfo.isExecutable();
f.size = fileInfo.size();
// FIXME: async / optimize the hashing
QFile input(fileInfo.absoluteFilePath());
if(!input.open(QIODevice::ReadOnly)) {
qCritical() << "Folder inspection: Failed to open file:" << fileInfo.absoluteFilePath();
out.valid = false;
break;
}
f.hash = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Sha1).toHex().constData();
out.addFile(relPath, f);
}
else {
// Something else... oh my
qCritical() << "Folder inspection: Unknown filesystem object:" << fileInfo.absoluteFilePath();
out.valid = false;
break;
}
}
out.folders.insert(Path("."));
out.valid = true;
return out;
}
namespace {
struct shallow_first_sort
{
bool operator()(const Path &lhs, const Path &rhs) const
{
auto lhs_depth = lhs.length();
auto rhs_depth = rhs.length();
if(lhs_depth < rhs_depth)
{
return true;
}
else if(lhs_depth == rhs_depth)
{
if(lhs < rhs)
{
return true;
}
}
return false;
}
};
struct deep_first_sort
{
bool operator()(const Path &lhs, const Path &rhs) const
{
auto lhs_depth = lhs.length();
auto rhs_depth = rhs.length();
if(lhs_depth > rhs_depth)
{
return true;
}
else if(lhs_depth == rhs_depth)
{
if(lhs < rhs)
{
return true;
}
}
return false;
}
};
}
UpdateOperations UpdateOperations::resolve(const Package& from, const Package& to)
{
UpdateOperations out;
if(!from.valid || !to.valid) {
out.valid = false;
return out;
}
// Files
for(auto iter = from.files.begin(); iter != from.files.end(); iter++) {
const auto &current_hash = iter->second.hash;
const auto &current_executable = iter->second.executable;
const auto &path = iter->first;
auto iter2 = to.files.find(path);
if(iter2 == to.files.end()) {
// removed
out.deletes.push_back(path);
continue;
}
auto new_hash = iter2->second.hash;
auto new_executable = iter2->second.executable;
if (current_hash != new_hash) {
out.deletes.push_back(path);
out.downloads.emplace(
std::pair<Path, FileDownload>{
path,
FileDownload(to.sources.at(iter2->second.hash), iter2->second.executable)
}
);
}
else if (current_executable != new_executable) {
out.executable_fixes[path] = new_executable;
}
}
for(auto iter = to.files.begin(); iter != to.files.end(); iter++) {
auto path = iter->first;
if(!from.files.count(path)) {
out.downloads.emplace(
std::pair<Path, FileDownload>{
path,
FileDownload(to.sources.at(iter->second.hash), iter->second.executable)
}
);
}
}
// Folders
std::set<Path, deep_first_sort> remove_folders;
std::set<Path, shallow_first_sort> make_folders;
for(auto from_path: from.folders) {
auto iter = to.folders.find(from_path);
if(iter == to.folders.end()) {
remove_folders.insert(from_path);
}
}
for(auto & rmdir: remove_folders) {
out.rmdirs.push_back(rmdir);
}
for(auto to_path: to.folders) {
auto iter = from.folders.find(to_path);
if(iter == from.folders.end()) {
make_folders.insert(to_path);
}
}
for(auto & mkdir: make_folders) {
out.mkdirs.push_back(mkdir);
}
// Symlinks
for(auto iter = from.symlinks.begin(); iter != from.symlinks.end(); iter++) {
const auto &current_target = iter->second;
const auto &path = iter->first;
auto iter2 = to.symlinks.find(path);
if(iter2 == to.symlinks.end()) {
// removed
out.deletes.push_back(path);
continue;
}
const auto &new_target = iter2->second;
if (current_target != new_target) {
out.deletes.push_back(path);
out.mklinks[path] = iter2->second;
}
}
for(auto iter = to.symlinks.begin(); iter != to.symlinks.end(); iter++) {
auto path = iter->first;
if(!from.symlinks.count(path)) {
out.mklinks[path] = iter->second;
}
}
out.valid = true;
return out;
}
}

View File

@ -0,0 +1,169 @@
#pragma once
#include <QString>
#include <map>
#include <set>
#include <QStringList>
#include "tasks/Task.h"
#include "multimc_logic_export.h"
namespace mojang_files {
using Hash = QString;
extern const Hash empty_hash;
// simple-ish path implementation. assumes always relative and does not allow '..' entries
class MULTIMC_LOGIC_EXPORT Path
{
public:
using parts_type = QStringList;
Path() = default;
Path(QString string) {
auto parts_in = string.split('/');
for(auto & part: parts_in) {
if(part.isEmpty() || part == ".") {
continue;
}
if(part == "..") {
if(parts.size()) {
parts.pop_back();
}
continue;
}
parts.push_back(part);
}
}
bool has_parent_path() const
{
return parts.size() > 0;
}
Path parent_path() const
{
if (parts.empty())
return Path();
return Path(parts.begin(), std::prev(parts.end()));
}
bool empty() const
{
return parts.empty();
}
int length() const
{
return parts.length();
}
bool operator==(const Path & rhs) const {
return parts == rhs.parts;
}
bool operator!=(const Path & rhs) const {
return parts != rhs.parts;
}
inline bool operator<(const Path& rhs) const
{
return compare(rhs) < 0;
}
parts_type::const_iterator begin() const
{
return parts.begin();
}
parts_type::const_iterator end() const
{
return parts.end();
}
QString toString() const {
return parts.join("/");
}
private:
Path(const parts_type::const_iterator & start, const parts_type::const_iterator & end) {
parts = QStringList(start, end);
}
int compare(const Path& p) const;
parts_type parts;
};
enum class Compression {
Raw,
Lzma,
Unknown
};
struct MULTIMC_LOGIC_EXPORT FileSource
{
Compression compression = Compression::Unknown;
Hash hash;
QString url;
std::size_t size = 0;
void upgrade(const FileSource & other) {
if(compression == Compression::Unknown || other.size < size) {
*this = other;
}
}
bool isBad() const {
return compression == Compression::Unknown;
}
};
struct MULTIMC_LOGIC_EXPORT File
{
Hash hash;
bool executable;
std::uint64_t size = 0;
};
struct MULTIMC_LOGIC_EXPORT Package {
static Package fromInspectedFolder(const QString &folderPath);
static Package fromManifestFile(const QString &path);
static Package fromManifestContents(const QByteArray& contents);
explicit operator bool() const
{
return valid;
}
void addFolder(Path folder);
void addFile(const Path & path, const File & file);
void addLink(const Path & path, const Path & target);
void addSource(const FileSource & source);
std::map<Hash, FileSource> sources;
bool valid = true;
std::set<Path> folders;
std::map<Path, File> files;
std::map<Path, Path> symlinks;
};
struct MULTIMC_LOGIC_EXPORT FileDownload : FileSource
{
FileDownload(const FileSource& source, bool executable) {
static_cast<FileSource &> (*this) = source;
this->executable = executable;
}
bool executable = false;
};
struct MULTIMC_LOGIC_EXPORT UpdateOperations {
static UpdateOperations resolve(const Package & from, const Package & to);
bool valid = false;
std::vector<Path> deletes;
std::vector<Path> rmdirs;
std::vector<Path> mkdirs;
std::map<Path, FileDownload> downloads;
std::map<Path, Path> mklinks;
std::map<Path, bool> executable_fixes;
};
}

View File

@ -0,0 +1,333 @@
#include <QTest>
#include <QDebug>
#include "TestUtil.h"
#include "mojang/PackageManifest.h"
using namespace mojang_files;
QDebug operator<<(QDebug debug, const Path &path)
{
debug << path.toString();
return debug;
}
class PackageManifestTest : public QObject
{
Q_OBJECT
private slots:
void test_parse();
void test_parse_file();
void test_inspect();
void test_diff();
void mkdir_deep();
void rmdir_deep();
void identical_file();
void changed_file();
void added_file();
void removed_file();
};
namespace {
QByteArray basic_manifest = R"END(
{
"files": {
"a/b.txt": {
"type": "file",
"downloads": {
"raw": {
"url": "http://dethware.org/b.txt",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"size": 0
}
},
"executable": true
},
"a/b/c": {
"type": "directory"
},
"a/b/c.txt": {
"type": "link",
"target": "../b.txt"
}
}
}
)END";
}
void PackageManifestTest::test_parse()
{
auto manifest = Package::fromManifestContents(basic_manifest);
QCOMPARE(manifest.valid, true);
QCOMPARE(manifest.files.size(), 1);
QCOMPARE(manifest.files.count(Path("a/b.txt")), 1);
auto &file = manifest.files[Path("a/b.txt")];
QCOMPARE(file.executable, true);
QCOMPARE(file.hash, "da39a3ee5e6b4b0d3255bfef95601890afd80709");
QCOMPARE(file.size, 0);
QCOMPARE(manifest.folders.size(), 4);
QCOMPARE(manifest.folders.count(Path(".")), 1);
QCOMPARE(manifest.folders.count(Path("a")), 1);
QCOMPARE(manifest.folders.count(Path("a/b")), 1);
QCOMPARE(manifest.folders.count(Path("a/b/c")), 1);
QCOMPARE(manifest.symlinks.size(), 1);
auto symlinkPath = Path("a/b/c.txt");
QCOMPARE(manifest.symlinks.count(symlinkPath), 1);
auto &symlink = manifest.symlinks[symlinkPath];
QCOMPARE(symlink, Path("../b.txt"));
QCOMPARE(manifest.sources.size(), 1);
}
void PackageManifestTest::test_parse_file() {
auto path = QFINDTESTDATA("testdata/1.8.0_202-x64.json");
auto manifest = Package::fromManifestFile(path);
QCOMPARE(manifest.valid, true);
}
void PackageManifestTest::test_inspect() {
auto path = QFINDTESTDATA("testdata/inspect/");
auto manifest = Package::fromInspectedFolder(path);
QCOMPARE(manifest.valid, true);
QCOMPARE(manifest.files.size(), 1);
QCOMPARE(manifest.files.count(Path("a/b.txt")), 1);
auto &file = manifest.files[Path("a/b.txt")];
QCOMPARE(file.executable, true);
QCOMPARE(file.hash, "da39a3ee5e6b4b0d3255bfef95601890afd80709");
QCOMPARE(file.size, 0);
QCOMPARE(manifest.folders.size(), 3);
QCOMPARE(manifest.folders.count(Path(".")), 1);
QCOMPARE(manifest.folders.count(Path("a")), 1);
QCOMPARE(manifest.folders.count(Path("a/b")), 1);
QCOMPARE(manifest.symlinks.size(), 1);
}
void PackageManifestTest::test_diff() {
auto path = QFINDTESTDATA("testdata/inspect/");
auto from = Package::fromInspectedFolder(path);
auto to = Package::fromManifestContents(basic_manifest);
auto operations = UpdateOperations::resolve(from, to);
QCOMPARE(operations.valid, true);
QCOMPARE(operations.mkdirs.size(), 1);
QCOMPARE(operations.mkdirs[0], Path("a/b/c"));
QCOMPARE(operations.rmdirs.size(), 0);
QCOMPARE(operations.deletes.size(), 1);
QCOMPARE(operations.deletes[0], Path("a/b/b.txt"));
QCOMPARE(operations.downloads.size(), 0);
QCOMPARE(operations.mklinks.size(), 1);
QCOMPARE(operations.mklinks.count(Path("a/b/c.txt")), 1);
QCOMPARE(operations.mklinks[Path("a/b/c.txt")], Path("../b.txt"));
}
void PackageManifestTest::mkdir_deep() {
Package from;
auto to = Package::fromManifestContents(R"END(
{
"files": {
"a/b/c/d/e": {
"type": "directory"
}
}
}
)END");
auto operations = UpdateOperations::resolve(from, to);
QCOMPARE(operations.deletes.size(), 0);
QCOMPARE(operations.rmdirs.size(), 0);
QCOMPARE(operations.mkdirs.size(), 6);
QCOMPARE(operations.mkdirs[0], Path("."));
QCOMPARE(operations.mkdirs[1], Path("a"));
QCOMPARE(operations.mkdirs[2], Path("a/b"));
QCOMPARE(operations.mkdirs[3], Path("a/b/c"));
QCOMPARE(operations.mkdirs[4], Path("a/b/c/d"));
QCOMPARE(operations.mkdirs[5], Path("a/b/c/d/e"));
QCOMPARE(operations.downloads.size(), 0);
QCOMPARE(operations.mklinks.size(), 0);
QCOMPARE(operations.executable_fixes.size(), 0);
}
void PackageManifestTest::rmdir_deep() {
Package to;
auto from = Package::fromManifestContents(R"END(
{
"files": {
"a/b/c/d/e": {
"type": "directory"
}
}
}
)END");
auto operations = UpdateOperations::resolve(from, to);
QCOMPARE(operations.deletes.size(), 0);
QCOMPARE(operations.rmdirs.size(), 6);
QCOMPARE(operations.rmdirs[0], Path("a/b/c/d/e"));
QCOMPARE(operations.rmdirs[1], Path("a/b/c/d"));
QCOMPARE(operations.rmdirs[2], Path("a/b/c"));
QCOMPARE(operations.rmdirs[3], Path("a/b"));
QCOMPARE(operations.rmdirs[4], Path("a"));
QCOMPARE(operations.rmdirs[5], Path("."));
QCOMPARE(operations.mkdirs.size(), 0);
QCOMPARE(operations.downloads.size(), 0);
QCOMPARE(operations.mklinks.size(), 0);
QCOMPARE(operations.executable_fixes.size(), 0);
}
void PackageManifestTest::identical_file() {
QByteArray manifest = R"END(
{
"files": {
"a/b/c/d/empty.txt": {
"type": "file",
"downloads": {
"raw": {
"url": "http://dethware.org/empty.txt",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"size": 0
}
},
"executable": false
}
}
}
)END";
auto from = Package::fromManifestContents(manifest);
auto to = Package::fromManifestContents(manifest);
auto operations = UpdateOperations::resolve(from, to);
QCOMPARE(operations.deletes.size(), 0);
QCOMPARE(operations.rmdirs.size(), 0);
QCOMPARE(operations.mkdirs.size(), 0);
QCOMPARE(operations.downloads.size(), 0);
QCOMPARE(operations.mklinks.size(), 0);
QCOMPARE(operations.executable_fixes.size(), 0);
}
void PackageManifestTest::changed_file() {
auto from = Package::fromManifestContents(R"END(
{
"files": {
"a/b/c/d/file": {
"type": "file",
"downloads": {
"raw": {
"url": "http://dethware.org/empty.txt",
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
"size": 0
}
},
"executable": false
}
}
}
)END");
auto to = Package::fromManifestContents(R"END(
{
"files": {
"a/b/c/d/file": {
"type": "file",
"downloads": {
"raw": {
"url": "http://dethware.org/space.txt",
"sha1": "dd122581c8cd44d0227f9c305581ffcb4b6f1b46",
"size": 1
}
},
"executable": false
}
}
}
)END");
auto operations = UpdateOperations::resolve(from, to);
QCOMPARE(operations.deletes.size(), 1);
QCOMPARE(operations.deletes[0], Path("a/b/c/d/file"));
QCOMPARE(operations.rmdirs.size(), 0);
QCOMPARE(operations.mkdirs.size(), 0);
QCOMPARE(operations.downloads.size(), 1);
QCOMPARE(operations.mklinks.size(), 0);
QCOMPARE(operations.executable_fixes.size(), 0);
}
void PackageManifestTest::added_file() {
auto from = Package::fromManifestContents(R"END(
{
"files": {
"a/b/c/d": {
"type": "directory"
}
}
}
)END");
auto to = Package::fromManifestContents(R"END(
{
"files": {
"a/b/c/d/file": {
"type": "file",
"downloads": {
"raw": {
"url": "http://dethware.org/space.txt",
"sha1": "dd122581c8cd44d0227f9c305581ffcb4b6f1b46",
"size": 1
}
},
"executable": false
}
}
}
)END");
auto operations = UpdateOperations::resolve(from, to);
QCOMPARE(operations.deletes.size(), 0);
QCOMPARE(operations.rmdirs.size(), 0);
QCOMPARE(operations.mkdirs.size(), 0);
QCOMPARE(operations.downloads.size(), 1);
QCOMPARE(operations.mklinks.size(), 0);
QCOMPARE(operations.executable_fixes.size(), 0);
}
void PackageManifestTest::removed_file() {
auto from = Package::fromManifestContents(R"END(
{
"files": {
"a/b/c/d/file": {
"type": "file",
"downloads": {
"raw": {
"url": "http://dethware.org/space.txt",
"sha1": "dd122581c8cd44d0227f9c305581ffcb4b6f1b46",
"size": 1
}
},
"executable": false
}
}
}
)END");
auto to = Package::fromManifestContents(R"END(
{
"files": {
"a/b/c/d": {
"type": "directory"
}
}
}
)END");
auto operations = UpdateOperations::resolve(from, to);
QCOMPARE(operations.deletes.size(), 1);
QCOMPARE(operations.deletes[0], Path("a/b/c/d/file"));
QCOMPARE(operations.rmdirs.size(), 0);
QCOMPARE(operations.mkdirs.size(), 0);
QCOMPARE(operations.downloads.size(), 0);
QCOMPARE(operations.mklinks.size(), 0);
QCOMPARE(operations.executable_fixes.size(), 0);
}
QTEST_GUILESS_MAIN(PackageManifestTest)
#include "PackageManifest_test.moc"

Binary file not shown.

0
api/logic/mojang/testdata/inspect/a/b.txt vendored Executable file
View File

View File

@ -0,0 +1 @@
../b.txt