fix: intelegent recursive links & symlink follow on export

Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com>
This commit is contained in:
Rachel Powers 2023-02-09 19:48:40 -07:00
parent bc8336a4b1
commit 2837236d81
8 changed files with 278 additions and 22 deletions

View File

@ -242,6 +242,14 @@ bool copy::operator()(const QString& offset, bool dryRun)
return err.value() == 0; return err.value() == 0;
} }
/// qDebug print support for the LinkPair struct
QDebug operator<<(QDebug debug, const LinkPair& lp)
{
QDebugStateSaver saver(debug);
debug.nospace() << "LinkPair{ src: " << lp.src << " , dst: " << lp.dst << " }";
return debug;
}
bool create_link::operator()(const QString& offset, bool dryRun) bool create_link::operator()(const QString& offset, bool dryRun)
{ {
@ -265,7 +273,7 @@ bool create_link::operator()(const QString& offset, bool dryRun)
* @param offset subdirectory form src to link to dest * @param offset subdirectory form src to link to dest
* @return if there was an error during the attempt to link * @return if there was an error during the attempt to link
*/ */
void create_link::make_link_list( const QString& offset) void create_link::make_link_list(const QString& offset)
{ {
for (auto pair : m_path_pairs) { for (auto pair : m_path_pairs) {
const QString& srcPath = pair.src; const QString& srcPath = pair.src;
@ -297,14 +305,26 @@ void create_link::make_link_list( const QString& offset)
link_file(src, ""); link_file(src, "");
} else { } else {
if (m_debug) if (m_debug)
qDebug() << "linking recursivly:" << src << "to" << dst; qDebug() << "linking recursivly:" << src << "to" << dst << "max_depth:" << m_max_depth;
QDir src_dir(src); QDir src_dir(src);
QDirIterator source_it(src, QDir::Filter::Files | QDir::Filter::Hidden, QDirIterator::Subdirectories); QDirIterator source_it(src, QDir::Filter::Files | QDir::Filter::Hidden, QDirIterator::Subdirectories);
QStringList linkedPaths;
while (source_it.hasNext()) { while (source_it.hasNext()) {
auto src_path = source_it.next(); auto src_path = source_it.next();
auto relative_path = src_dir.relativeFilePath(src_path); auto relative_path = src_dir.relativeFilePath(src_path);
if (m_max_depth >= 0 && PathDepth(relative_path) > m_max_depth){
relative_path = PathTruncate(relative_path, m_max_depth);
src_path = src_dir.filePath(relative_path);
if (linkedPaths.contains(src_path)) {
continue;
}
}
linkedPaths.append(src_path);
link_file(src_path, relative_path); link_file(src_path, relative_path);
} }
} }
@ -556,11 +576,49 @@ QString PathCombine(const QString& path1, const QString& path2, const QString& p
return PathCombine(PathCombine(path1, path2, path3), path4); return PathCombine(PathCombine(path1, path2, path3), path4);
} }
QString AbsolutePath(QString path) QString AbsolutePath(const QString& path)
{ {
return QFileInfo(path).absolutePath(); return QFileInfo(path).absolutePath();
} }
int PathDepth(const QString& path)
{
if (path.isEmpty()) return 0;
QFileInfo info(path);
auto parts = QDir::toNativeSeparators(info.path()).split(QDir::separator(), Qt::SkipEmptyParts);
int numParts = QDir::toNativeSeparators(info.path()).split(QDir::separator(), Qt::SkipEmptyParts).length();
numParts -= parts.count(".");
numParts -= parts.count("..") * 2;
return numParts;
}
QString PathTruncate(const QString& path, int depth)
{
if (path.isEmpty() || (depth < 0) ) return "";
QString trunc = QFileInfo(path).path();
if (PathDepth(trunc) > depth ) {
return PathTruncate(trunc, depth);
}
auto parts = QDir::toNativeSeparators(trunc).split(QDir::separator(), Qt::SkipEmptyParts);
if (parts.startsWith(".") && !path.startsWith(".")) {
parts.removeFirst();
}
if (path.startsWith(QDir::separator())) {
parts.prepend("");
}
trunc = parts.join(QDir::separator());
return trunc;
}
QString ResolveExecutable(QString path) QString ResolveExecutable(QString path)
{ {
if (path.isEmpty()) { if (path.isEmpty()) {

View File

@ -200,6 +200,11 @@ class create_link : public QObject {
m_recursive = recursive; m_recursive = recursive;
return *this; return *this;
} }
create_link& setMaxDepth(int depth)
{
m_max_depth = depth;
return *this;
}
create_link& debug(bool d) create_link& debug(bool d)
{ {
m_debug = d; m_debug = d;
@ -239,6 +244,9 @@ class create_link : public QObject {
bool m_whitelist = false; bool m_whitelist = false;
bool m_recursive = true; bool m_recursive = true;
/// @brief >= -1 = infinite, 0 = link files at src/* to dest/*, 1 = link files at src/*/* to dest/*/*, etc.
int m_max_depth = -1;
QList<LinkPair> m_path_pairs; QList<LinkPair> m_path_pairs;
QList<LinkResult> m_path_results; QList<LinkResult> m_path_results;
QList<LinkPair> m_links_to_make; QList<LinkPair> m_links_to_make;
@ -272,7 +280,25 @@ QString PathCombine(const QString& path1, const QString& path2);
QString PathCombine(const QString& path1, const QString& path2, const QString& path3); QString PathCombine(const QString& path1, const QString& path2, const QString& path3);
QString PathCombine(const QString& path1, const QString& path2, const QString& path3, const QString& path4); QString PathCombine(const QString& path1, const QString& path2, const QString& path3, const QString& path4);
QString AbsolutePath(QString path); QString AbsolutePath(const QString& path);
/**
* @brief depth of path. "foo.txt" -> 0 , "bar/foo.txt" -> 1, /baz/bar/foo.txt -> 2, etc.
*
* @param path path to measure
* @return int number of componants before base path
*/
int PathDepth(const QString& path);
/**
* @brief cut off segments of path untill it is a max of length depth
*
* @param path path to truncate
* @param depth max depth of new path
* @return QString truncated path
*/
QString PathTruncate(const QString& path, int depth);
/** /**
* Resolve an executable * Resolve an executable

View File

@ -16,8 +16,13 @@ bool InstanceCopyPrefs::allTrue() const
copyScreenshots; copyScreenshots;
} }
// Returns a single RegEx string of the selected folders/files to filter out (ex: ".minecraft/saves|.minecraft/server.dat") // Returns a single RegEx string of the selected folders/files to filter out (ex: ".minecraft/saves|.minecraft/server.dat")
QString InstanceCopyPrefs::getSelectedFiltersAsRegex() const QString InstanceCopyPrefs::getSelectedFiltersAsRegex() const
{
return getSelectedFiltersAsRegex({});
}
QString InstanceCopyPrefs::getSelectedFiltersAsRegex(const QStringList& additionalFilters) const
{ {
QStringList filters; QStringList filters;
@ -42,6 +47,10 @@ QString InstanceCopyPrefs::getSelectedFiltersAsRegex() const
if(!copyScreenshots) if(!copyScreenshots)
filters << "screenshots"; filters << "screenshots";
for (auto filter : additionalFilters) {
filters << filter;
}
// If we have any filters to add, join them as a single regex string to return: // If we have any filters to add, join them as a single regex string to return:
if (!filters.isEmpty()) { if (!filters.isEmpty()) {
const QString MC_ROOT = "[.]?minecraft/"; const QString MC_ROOT = "[.]?minecraft/";

View File

@ -10,6 +10,7 @@ struct InstanceCopyPrefs {
public: public:
[[nodiscard]] bool allTrue() const; [[nodiscard]] bool allTrue() const;
[[nodiscard]] QString getSelectedFiltersAsRegex() const; [[nodiscard]] QString getSelectedFiltersAsRegex() const;
[[nodiscard]] QString getSelectedFiltersAsRegex(const QStringList& additionalFilters) const;
// Getters // Getters
[[nodiscard]] bool isCopySavesEnabled() const; [[nodiscard]] bool isCopySavesEnabled() const;
[[nodiscard]] bool isKeepPlaytimeEnabled() const; [[nodiscard]] bool isKeepPlaytimeEnabled() const;

View File

@ -4,13 +4,14 @@
#include "NullInstance.h" #include "NullInstance.h"
#include "pathmatcher/RegexpMatcher.h" #include "pathmatcher/RegexpMatcher.h"
#include <QtConcurrentRun> #include <QtConcurrentRun>
#include <QDebug>
InstanceCopyTask::InstanceCopyTask(InstancePtr origInstance, const InstanceCopyPrefs& prefs) InstanceCopyTask::InstanceCopyTask(InstancePtr origInstance, const InstanceCopyPrefs& prefs)
{ {
m_origInstance = origInstance; m_origInstance = origInstance;
m_keepPlaytime = prefs.isKeepPlaytimeEnabled(); m_keepPlaytime = prefs.isKeepPlaytimeEnabled();
QString filters = prefs.getSelectedFiltersAsRegex();
m_useLinks = prefs.isUseSymLinksEnabled(); m_useLinks = prefs.isUseSymLinksEnabled();
@ -19,6 +20,14 @@ InstanceCopyTask::InstanceCopyTask(InstancePtr origInstance, const InstanceCopyP
m_copySaves = prefs.isLinkRecursivelyEnabled() && prefs.isDontLinkSavesEnabled() && prefs.isCopySavesEnabled(); m_copySaves = prefs.isLinkRecursivelyEnabled() && prefs.isDontLinkSavesEnabled() && prefs.isCopySavesEnabled();
m_useClone = prefs.isUseCloneEnabled(); m_useClone = prefs.isUseCloneEnabled();
QString filters = prefs.getSelectedFiltersAsRegex();
if (m_useLinks || m_useHardLinks) {
if (!filters.isEmpty()) filters += "|";
filters += "instance.cfg";
}
qDebug() << "CopyFilters:" << filters;
if (!filters.isEmpty()) if (!filters.isEmpty())
{ {
// Set regex filter: // Set regex filter:
@ -46,9 +55,10 @@ void InstanceCopyTask::executeTask()
folderClone.matcher(m_matcher.get()); folderClone.matcher(m_matcher.get());
return folderClone(); return folderClone();
} else if (m_useLinks) { } else if (m_useLinks || m_useHardLinks) {
FS::create_link folderLink(m_origInstance->instanceRoot(), m_stagingPath); FS::create_link folderLink(m_origInstance->instanceRoot(), m_stagingPath);
folderLink.linkRecursively(m_linkRecursively).useHardLinks(m_useHardLinks).matcher(m_matcher.get()); int depth = m_linkRecursively ? -1 : 0; // we need to at least link the top level instead of the instance folder
folderLink.linkRecursively(true).setMaxDepth(depth).useHardLinks(m_useHardLinks).matcher(m_matcher.get());
bool there_were_errors = false; bool there_were_errors = false;

View File

@ -102,8 +102,13 @@ bool MMCZip::compressDirFiles(QuaZip *zip, QString dir, QFileInfoList files, boo
for (auto e : files) { for (auto e : files) {
auto filePath = directory.relativeFilePath(e.absoluteFilePath()); auto filePath = directory.relativeFilePath(e.absoluteFilePath());
auto srcPath = e.absoluteFilePath(); auto srcPath = e.absoluteFilePath();
if (followSymlinks) if (followSymlinks) {
if (e.isSymLink()) {
srcPath = e.symLinkTarget();
} else {
srcPath = e.canonicalFilePath(); srcPath = e.canonicalFilePath();
}
}
if( !JlCompress::compressFile(zip, srcPath, filePath)) return false; if( !JlCompress::compressFile(zip, srcPath, filePath)) return false;
} }
@ -119,7 +124,7 @@ bool MMCZip::compressDirFiles(QString fileCompressed, QString dir, QFileInfoList
return false; return false;
} }
auto result = compressDirFiles(&zip, dir, files); auto result = compressDirFiles(&zip, dir, files, followSymlinks);
zip.close(); zip.close();
if(zip.getZipError()!=0) { if(zip.getZipError()!=0) {

View File

@ -171,17 +171,18 @@ void CopyInstanceDialog::updateSelectAllCheckbox()
void CopyInstanceDialog::updateUseCloneCheckbox() void CopyInstanceDialog::updateUseCloneCheckbox()
{ {
ui->useCloneCheckbox->setEnabled(m_cloneSupported && !ui->linkFilesGroup->isChecked()); ui->useCloneCheckbox->setEnabled(m_cloneSupported && !ui->symbolicLinksCheckbox->isChecked() && !ui->hardLinksCheckbox->isChecked());
ui->useCloneCheckbox->setChecked(m_cloneSupported && m_selectedOptions.isUseCloneEnabled()); ui->useCloneCheckbox->setChecked(m_cloneSupported && m_selectedOptions.isUseCloneEnabled() && !ui->symbolicLinksCheckbox->isChecked() &&
!ui->hardLinksCheckbox->isChecked());
} }
void CopyInstanceDialog::updateLinkOptions() void CopyInstanceDialog::updateLinkOptions()
{ {
ui->symbolicLinksCheckbox->setEnabled(m_linkSupported && !ui->hardLinksCheckbox->isChecked()); ui->symbolicLinksCheckbox->setEnabled(m_linkSupported && !ui->hardLinksCheckbox->isChecked() && !ui->useCloneCheckbox->isChecked());
ui->hardLinksCheckbox->setEnabled(m_linkSupported && !ui->symbolicLinksCheckbox->isChecked()); ui->hardLinksCheckbox->setEnabled(m_linkSupported && !ui->symbolicLinksCheckbox->isChecked() && !ui->useCloneCheckbox->isChecked());
ui->symbolicLinksCheckbox->setChecked(m_linkSupported && m_selectedOptions.isUseSymLinksEnabled()); ui->symbolicLinksCheckbox->setChecked(m_linkSupported && m_selectedOptions.isUseSymLinksEnabled() && !ui->useCloneCheckbox->isChecked());
ui->hardLinksCheckbox->setChecked(m_linkSupported && m_selectedOptions.isUseHardLinksEnabled()); ui->hardLinksCheckbox->setChecked(m_linkSupported && m_selectedOptions.isUseHardLinksEnabled() && !ui->useCloneCheckbox->isChecked());
bool linksInUse = (ui->symbolicLinksCheckbox->isChecked() || ui->hardLinksCheckbox->isChecked()); bool linksInUse = (ui->symbolicLinksCheckbox->isChecked() || ui->hardLinksCheckbox->isChecked());
ui->recursiveLinkCheckbox->setEnabled(m_linkSupported && linksInUse && !ui->hardLinksCheckbox->isChecked()); ui->recursiveLinkCheckbox->setEnabled(m_linkSupported && linksInUse && !ui->hardLinksCheckbox->isChecked());
@ -278,16 +279,14 @@ void CopyInstanceDialog::on_hardLinksCheckbox_stateChanged(int state)
if (state == Qt::Checked && !ui->recursiveLinkCheckbox->isChecked()) { if (state == Qt::Checked && !ui->recursiveLinkCheckbox->isChecked()) {
ui->recursiveLinkCheckbox->setChecked(true); ui->recursiveLinkCheckbox->setChecked(true);
} }
updateUseCloneCheckbox();
updateLinkOptions(); updateLinkOptions();
} }
void CopyInstanceDialog::on_recursiveLinkCheckbox_stateChanged(int state) void CopyInstanceDialog::on_recursiveLinkCheckbox_stateChanged(int state)
{ {
m_selectedOptions.enableLinkRecursively(state == Qt::Checked); m_selectedOptions.enableLinkRecursively(state == Qt::Checked);
if (state != Qt::Checked) { updateLinkOptions();
ui->hardLinksCheckbox->setChecked(false);
ui->dontLinkSavesCheckbox->setChecked(false);
}
} }
@ -299,6 +298,6 @@ void CopyInstanceDialog::on_dontLinkSavesCheckbox_stateChanged(int state)
void CopyInstanceDialog::on_useCloneCheckbox_stateChanged(int state) void CopyInstanceDialog::on_useCloneCheckbox_stateChanged(int state)
{ {
m_selectedOptions.enableUseClone(m_cloneSupported && (state == Qt::Checked)); m_selectedOptions.enableUseClone(m_cloneSupported && (state == Qt::Checked));
ui->linkFilesGroup->setEnabled(!m_selectedOptions.isUseCloneEnabled());
updateUseCloneCheckbox(); updateUseCloneCheckbox();
updateLinkOptions();
} }

View File

@ -57,6 +57,11 @@ class LinkTask : public Task {
m_lnk->whitelist(b); m_lnk->whitelist(b);
} }
void setMaxDepth(int depth)
{
m_lnk->setMaxDepth(depth);
}
private: private:
void executeTask() override void executeTask() override
{ {
@ -630,6 +635,149 @@ slots:
QVERIFY(target_dir.entryList(filter).contains("pack.mcmeta")); QVERIFY(target_dir.entryList(filter).contains("pack.mcmeta"));
} }
} }
void test_link_with_max_depth()
{
QString folder = QFINDTESTDATA("testdata/FileSystem/test_folder");
auto f = [&folder, this]()
{
QTemporaryDir tempDir;
tempDir.setAutoRemove(true);
qDebug() << "From:" << folder << "To:" << tempDir.path();
QDir target_dir(FS::PathCombine(tempDir.path(), "test_folder"));
qDebug() << tempDir.path();
qDebug() << target_dir.path();
LinkTask lnk_tsk(folder, target_dir.path());
lnk_tsk.linkRecursively(true);
lnk_tsk.setMaxDepth(0);
QObject::connect(&lnk_tsk, &Task::finished, [&]{
QVERIFY2(lnk_tsk.wasSuccessful(), "Task finished but was not successful when it should have been.");
});
lnk_tsk.start();
QVERIFY2(QTest::qWaitFor([&]() {
return lnk_tsk.isFinished();
}, 100000), "Task didn't finish as it should.");
QVERIFY(!QFileInfo(target_dir.path()).isSymLink());
auto filter = QDir::Filter::Files | QDir::Filter::Dirs | QDir::Filter::Hidden;
for(auto entry: target_dir.entryList(filter))
{
qDebug() << entry;
if (entry == "." || entry == "..") continue;
QFileInfo entry_lnk_info(target_dir.filePath(entry));
QVERIFY(entry_lnk_info.isSymLink());
}
QFileInfo lnk_info(target_dir.path());
QVERIFY(lnk_info.exists());
QVERIFY(!lnk_info.isSymLink());
QVERIFY(target_dir.entryList().contains("pack.mcmeta"));
QVERIFY(target_dir.entryList().contains("assets"));
};
// first try variant without trailing /
QVERIFY(!folder.endsWith('/'));
f();
// then variant with trailing /
folder.append('/');
QVERIFY(folder.endsWith('/'));
f();
}
void test_link_with_no_max_depth()
{
QString folder = QFINDTESTDATA("testdata/FileSystem/test_folder");
auto f = [&folder]()
{
QTemporaryDir tempDir;
tempDir.setAutoRemove(true);
qDebug() << "From:" << folder << "To:" << tempDir.path();
QDir target_dir(FS::PathCombine(tempDir.path(), "test_folder"));
qDebug() << tempDir.path();
qDebug() << target_dir.path();
LinkTask lnk_tsk(folder, target_dir.path());
lnk_tsk.linkRecursively(true);
lnk_tsk.setMaxDepth(-1);
QObject::connect(&lnk_tsk, &Task::finished, [&]{
QVERIFY2(lnk_tsk.wasSuccessful(), "Task finished but was not successful when it should have been.");
});
lnk_tsk.start();
QVERIFY2(QTest::qWaitFor([&]() {
return lnk_tsk.isFinished();
}, 100000), "Task didn't finish as it should.");
std::function<void(QString)> verify_check = [&](QString check_path) {
QDir check_dir(check_path);
auto filter = QDir::Filter::Files | QDir::Filter::Dirs | QDir::Filter::Hidden;
for(auto entry: check_dir.entryList(filter))
{
QFileInfo entry_lnk_info(check_dir.filePath(entry));
qDebug() << entry << check_dir.filePath(entry);
if (!entry_lnk_info.isDir()){
QVERIFY(entry_lnk_info.isSymLink());
} else if (entry != "." && entry != "..") {
qDebug() << "Decending tree to verify symlinks:" << check_dir.filePath(entry);
verify_check(entry_lnk_info.filePath());
}
}
};
verify_check(target_dir.path());
QFileInfo lnk_info(target_dir.path());
QVERIFY(lnk_info.exists());
QVERIFY(target_dir.entryList().contains("pack.mcmeta"));
QVERIFY(target_dir.entryList().contains("assets"));
};
// first try variant without trailing /
QVERIFY(!folder.endsWith('/'));
f();
// then variant with trailing /
folder.append('/');
QVERIFY(folder.endsWith('/'));
f();
}
void test_path_depth() {
QCOMPARE_EQ(FS::PathDepth(""), 0);
QCOMPARE_EQ(FS::PathDepth("."), 0);
QCOMPARE_EQ(FS::PathDepth("foo.txt"), 0);
QCOMPARE_EQ(FS::PathDepth("./foo.txt"), 0);
QCOMPARE_EQ(FS::PathDepth("./bar/foo.txt"), 1);
QCOMPARE_EQ(FS::PathDepth("../bar/foo.txt"), 0);
QCOMPARE_EQ(FS::PathDepth("/bar/foo.txt"), 1);
QCOMPARE_EQ(FS::PathDepth("baz/bar/foo.txt"), 2);
QCOMPARE_EQ(FS::PathDepth("/baz/bar/foo.txt"), 2);
QCOMPARE_EQ(FS::PathDepth("./baz/bar/foo.txt"), 2);
QCOMPARE_EQ(FS::PathDepth("/baz/../bar/foo.txt"), 1);
}
void test_path_trunc() {
QCOMPARE_EQ(FS::PathTruncate("", 0), "");
QCOMPARE_EQ(FS::PathTruncate("foo.txt", 0), "");
QCOMPARE_EQ(FS::PathTruncate("foo.txt", 1), "");
QCOMPARE_EQ(FS::PathTruncate("./bar/foo.txt", 0), "./bar");
QCOMPARE_EQ(FS::PathTruncate("./bar/foo.txt", 1), "./bar");
QCOMPARE_EQ(FS::PathTruncate("/bar/foo.txt", 1), "/bar");
QCOMPARE_EQ(FS::PathTruncate("bar/foo.txt", 1), "bar");
QCOMPARE_EQ(FS::PathTruncate("baz/bar/foo.txt", 2), "baz/bar");
}
}; };
QTEST_GUILESS_MAIN(FileSystemTest) QTEST_GUILESS_MAIN(FileSystemTest)