Split archives at size limits (#40)

* Split archives * Add some comments and exclude separators from valid mods * Allow creating override file for each archive --------- Co-authored-by: Liderate <[email protected]>
ModOrganizer2 · Jul 5, 2024 · 2a9462b · 2a9462b
1 parent 2ee77e4
commit 2a9462b
Show file tree

Hide file tree

Showing 17 changed files with 155 additions and 57 deletions.
diff --git a/src/ArchiveNameService.cpp b/src/ArchiveNameService.cpp
@@ -2,6 +2,8 @@
 
 #include "NexusId.h"
 
+#include <QFileInfo>
+
 namespace BsaPacker
 {
 	ArchiveNameService::ArchiveNameService(const IModContext* modContext)
@@ -31,7 +33,9 @@ namespace BsaPacker
 
 	QString ArchiveNameService::GetArchiveFullPath(const bsa_archive_type_e type, const IModDto* modDto) const
 	{
-		return QDir::toNativeSeparators(modDto->Directory() + '/' + modDto->ArchiveName() + this->Infix(type) + this->GetFileExtension());
+		const QString& pathNoExt(QDir::toNativeSeparators(modDto->Directory() + '/' + modDto->ArchiveName() + this->Infix(type)));
+		const QString& suffix = this->Suffix(pathNoExt);
+		return QDir::toNativeSeparators(pathNoExt + suffix + this->GetFileExtension());
 	}
 
 	QString ArchiveNameService::Infix(const bsa_archive_type_e type) const
@@ -52,4 +56,20 @@ namespace BsaPacker
 			return QString();
 		};
 	}
+
+	// gets the number to append when there are multiple archives
+	// a way to avoid overwriting any existing files
+	QString ArchiveNameService::Suffix(const QString& pathNoExt) const {
+		int archiveIndex = 0;
+		const QString& fileExt = this->GetFileExtension();
+		QFileInfo fileInfo(pathNoExt + fileExt);
+		while (fileInfo.exists()) {
+			++archiveIndex;
+			fileInfo.setFile(pathNoExt + QString::number(archiveIndex) + fileExt);
+		}
+		if (archiveIndex != 0) {
+			return QString::number(archiveIndex);
+		}
+		return QString();
+	}
 }
diff --git a/src/ArchiveNameService.h b/src/ArchiveNameService.h
@@ -15,9 +15,9 @@ namespace BsaPacker
 		QString GetArchiveFullPath(bsa_archive_type_e type, const IModDto* modDto) const override;
 		QString GetFileExtension() const override;
 		QString Infix(bsa_archive_type_e type) const override;
+		QString Suffix(const QString& pathNoExt) const override;
 	private:
 		const IModContext* m_ModContext = nullptr;
-
 	};
 }
 

diff --git a/src/BsaPackerWorker.cpp b/src/BsaPackerWorker.cpp
@@ -29,25 +29,34 @@ namespace BsaPacker
 
 	void BsaPackerWorker::DoWork() const
 	{
+		QStringList createdArchives;
 		const std::unique_ptr<IModDto> modDto = this->m_ModDtoFactory->Create(); // handles PackerDialog and validation, implements Null Object pattern
 		const std::vector<bsa_archive_type_e> types = this->m_ArchiveBuilderFactory->GetArchiveTypes(modDto.get());
 		for (auto&& type : types) {
 			const std::unique_ptr<IArchiveBuilder> builder = this->m_ArchiveBuilderFactory->Create(type, modDto.get());
 			ArchiveBuildDirector director(this->m_SettingsService, builder.get());
 			director.Construct(); // must check if cancelled
-			const std::unique_ptr<libbsarch::bs_archive_auto> archive = builder->getArchive();
-			if (archive) {
-				const QString& archiveFullPath = this->m_ArchiveNameService->GetArchiveFullPath(type, modDto.get());
-				bool res = this->m_ArchiveAutoService->CreateBSA(archive.get(), archiveFullPath, type);
-				if (res)
-					QMessageBox::information(nullptr, "", QObject::tr("Created") + " \"" + archiveFullPath + "\"");
+			const std::vector<std::unique_ptr<libbsarch::bs_archive_auto>> archives = builder->getArchives();
+			for (const auto& archive : archives) {
+				if (archive) {
+					const QFileInfo fileInfo(this->m_ArchiveNameService->GetArchiveFullPath(type, modDto.get()));
+					bool res = this->m_ArchiveAutoService->CreateBSA(archive.get(), fileInfo.absoluteFilePath(), type);
+					if (res) {
+						createdArchives.append(fileInfo.baseName());
+					}
+				}
 			}
 		}
+
+		if (!createdArchives.isEmpty()) {
+			QMessageBox::information(nullptr, "",
+        QObject::tr("Created archive(s):") + "\n" + createdArchives.join(modDto->ArchiveExtension() +",\n") + modDto->ArchiveExtension());
+			this->m_OverrideFileService->CreateOverrideFile(modDto->NexusId(), modDto->Directory(), createdArchives);
+		}
+
 		const std::unique_ptr<IDummyPluginService> pluginService = this->m_DummyPluginServiceFactory->Create();
 		pluginService->CreatePlugin(modDto->Directory(), modDto->ArchiveName());
 
-		this->m_OverrideFileService->CreateOverrideFile(modDto->NexusId(), modDto->Directory(), modDto->ArchiveName());
-
 		if (!modDto->Directory().isEmpty()) {
 			this->m_HideLooseAssetService->HideLooseAssets(modDto->Directory());
 		}

diff --git a/src/GeneralArchiveBuilder.cpp b/src/GeneralArchiveBuilder.cpp
@@ -9,18 +9,22 @@ using namespace libbsarch;
 
 namespace BsaPacker
 {
+	// 2 GiB limit. This does not consider size after compression or share data
+	const qint64 GeneralArchiveBuilder::SIZE_LIMIT = (qint64)1024 * 1024 * 1024 * 2;
+
 	GeneralArchiveBuilder::GeneralArchiveBuilder(const IArchiveBuilderHelper* archiveBuilderHelper, const QDir& rootDir, const bsa_archive_type_t& type)
-		: m_ArchiveBuilderHelper(archiveBuilderHelper), m_RootDirectory(rootDir)
+		: m_ArchiveBuilderHelper(archiveBuilderHelper), m_RootDirectory(rootDir), m_ArchiveType(type)
 	{
 		this->m_Cancelled = false;
-		this->m_Archive = std::make_unique<libbsarch::bs_archive_auto>(type);
+		this->m_Archives.emplace_back(std::make_unique<libbsarch::bs_archive_auto>(this->m_ArchiveType));
 	}
 
 	uint32_t GeneralArchiveBuilder::setFiles()
 	{
 		uint32_t incompressibleFiles = 0;
 		uint32_t compressibleFiles = 0;
 		int count = 0;
+		qint64 currentSize = 0;
 		const auto& dirString = (this->m_RootDirectory.path() + '/').toStdWString();
 		const auto& rootDirFiles = this->m_ArchiveBuilderHelper->getRootDirectoryFilenames(dirString);
 		qDebug() << "root is: " << m_RootDirectory.path() + '/';
@@ -30,37 +34,50 @@ namespace BsaPacker
 			QApplication::processEvents();
 
 			if (this->m_Cancelled) {
-				this->m_Archive.reset();
+				for (auto& archive : this->m_Archives) {
+					archive.reset();
+				}
 				return 0;
 			}
 
-			const QString& filepath = iterator.next();
+			const QFileInfo& fileInfo = iterator.nextFileInfo();
+			const QString& filepath = fileInfo.absoluteFilePath();
 			const bool ignored = this->m_ArchiveBuilderHelper->isFileIgnorable(filepath.toStdWString(), rootDirFiles);
 
 			Q_EMIT this->valueChanged(++count);
 			if (ignored) {
 				continue;
 			}
 
+			currentSize += fileInfo.size();
+			if (currentSize > SIZE_LIMIT) {
+				currentSize = fileInfo.size();
+				this->m_Archives.back()->set_compressed(!static_cast<bool>(incompressibleFiles));
+				incompressibleFiles = 0;
+				compressibleFiles = 0;
+				this->m_Archives.emplace_back(std::make_unique<libbsarch::bs_archive_auto>(this->m_ArchiveType));
+				this->setShareData(true);
+			}
+
 			this->m_ArchiveBuilderHelper->isIncompressible(filepath.toStdWString()) ? ++incompressibleFiles : ++compressibleFiles;
 			auto fileBlob = disk_blob(
 				 dirString,
 				 filepath.toStdWString());
-			this->m_Archive->add_file_from_disk(fileBlob);
+			this->m_Archives.back()->add_file_from_disk(fileBlob);
 			qDebug() << "file is: " << filepath;
 		}
-		this->m_Archive->set_compressed(!static_cast<bool>(incompressibleFiles));
+		this->m_Archives.back()->set_compressed(!static_cast<bool>(incompressibleFiles));
 		return incompressibleFiles + compressibleFiles;
 	}
 
 	void GeneralArchiveBuilder::setShareData(const bool value)
 	{
-		this->m_Archive->set_share_data(value);
+		this->m_Archives.back()->set_share_data(value);
 	}
 
-	std::unique_ptr<libbsarch::bs_archive_auto> GeneralArchiveBuilder::getArchive()
+	std::vector<std::unique_ptr<libbsarch::bs_archive_auto>> GeneralArchiveBuilder::getArchives()
 	{
-		return std::move(this->m_Archive);
+		return std::move(this->m_Archives);
 	}
 
 	uint32_t GeneralArchiveBuilder::getFileCount() const

diff --git a/src/ModContext.cpp b/src/ModContext.cpp
@@ -59,7 +59,7 @@ namespace BsaPacker
 		const MOBase::IModList* const list = m_Organizer->modList();
 		const std::function<bool(const QString&)> modStateValid = [&](const QString& mod)
 		{
-			return list->state(mod) & MOBase::IModList::STATE_VALID;
+			return !mod.endsWith("_separator", Qt::CaseInsensitive) && (list->state(mod) & MOBase::IModList::STATE_VALID);
 		};
 		return QtConcurrent::blockingFiltered(list->allMods(), modStateValid);
 	}

diff --git a/src/NullArchiveBuilder.cpp b/src/NullArchiveBuilder.cpp
@@ -11,9 +11,9 @@ namespace BsaPacker
 	{
 	}
 
-	std::unique_ptr<libbsarch::bs_archive_auto> NullArchiveBuilder::getArchive()
+	std::vector<std::unique_ptr<libbsarch::bs_archive_auto>> NullArchiveBuilder::getArchives()
 	{
-		return nullptr;
+		return {};
 	}
 
 	uint32_t NullArchiveBuilder::getFileCount() const

diff --git a/src/OverrideFileService.cpp b/src/OverrideFileService.cpp
@@ -17,20 +17,28 @@ namespace BsaPacker {
 	// TODO: Add detection for Command Extender and JIP LN NVSE and warn if missing
 	bool OverrideFileService::CreateOverrideFile(const int nexusId,
 		const QString& modPath,
-		const QString& archiveNameBase) const {
+		const QStringList& archiveNames) const {
 
 		if (nexusId != FALLOUT_3_NEXUS_ID && nexusId != NEW_VEGAS_NEXUS_ID) {
 			return false;
 		}
 
 		if (MOBase::QuestionBoxMemory::query(QApplication::activeModalWidget(), "BSAPacker", "Create .override file?",
-			"Do you want to create an override file for the archive?",
+			"Do you want to create an override file for the archive(s)?",
 			QDialogButtonBox::No | QDialogButtonBox::Yes, QDialogButtonBox::No) & QDialogButtonBox::No) {
 			return false;
 		}
 
-		const QString& fileNameNoExtension = modPath + '/' + archiveNameBase;
-		const std::string& absoluteFileName = fileNameNoExtension.toStdString() + ".override";
-		return this->m_FileWriterService->Write(absoluteFileName, nullptr, 0);
+		bool res = true;
+		for (const QString& baseName : archiveNames) {
+			const QString& fileNameNoExtension = modPath + '/' + baseName;
+			const std::string& absoluteFileName = fileNameNoExtension.toStdString() + ".override";
+			if (!this->m_FileWriterService->Write(absoluteFileName, nullptr, 0)) {
+				qWarning() << "Failed to create" << absoluteFileName;
+				res = false;
+			}
+		}
+
+		return res;
 	}
 } // namespace BsaPacker
diff --git a/src/OverrideFileService.h b/src/OverrideFileService.h
@@ -11,7 +11,7 @@ namespace BsaPacker {
 		OverrideFileService(const IFileWriterService* fileWriterService);
 		bool CreateOverrideFile(const int nexusId,
 			const QString& modPath,
-			const QString& archiveNameBase) const override;
+			const QStringList& archiveNames) const override;
 
 	private:
 		const IFileWriterService* m_FileWriterService = nullptr;

diff --git a/src/TextureArchiveBuilder.cpp b/src/TextureArchiveBuilder.cpp
@@ -10,18 +10,23 @@ using namespace libbsarch;
 
 namespace BsaPacker
 {
+	// 4 GiB limit for the Fallout 4 Creation Kit. The game itself has no limit so could make an optional setting
+	// This does not consider size after compression or share data
+	const qint64 TextureArchiveBuilder::SIZE_LIMIT = (qint64)1024 * 1024 * 1024 * 4;
+
 	TextureArchiveBuilder::TextureArchiveBuilder(const IArchiveBuilderHelper* archiveBuilderHelper, const QDir& rootDir, const bsa_archive_type_t& type)
-		: m_ArchiveBuilderHelper(archiveBuilderHelper), m_RootDirectory(rootDir)
+		: m_ArchiveBuilderHelper(archiveBuilderHelper), m_RootDirectory(rootDir), m_ArchiveType(type)
 	{
 		this->m_Cancelled = false;
-		this->m_Archive = std::make_unique<libbsarch::bs_archive_auto>(type);
+		this->m_Archives.emplace_back(std::make_unique<libbsarch::bs_archive_auto>(this->m_ArchiveType));
 	}
 
 	uint32_t TextureArchiveBuilder::setFiles()
 	{
 		uint32_t incompressibleFiles = 0;
 		uint32_t compressibleFiles = 0;
 		int count = 0;
+		qint64 currentSize = 0;
 		const auto& dirString = (this->m_RootDirectory.path() + '/').toStdWString();
 		const auto& rootDirFiles = this->m_ArchiveBuilderHelper->getRootDirectoryFilenames(dirString);
 		qDebug() << "root is: " << m_RootDirectory.path() + '/';
@@ -31,38 +36,52 @@ namespace BsaPacker
 			QApplication::processEvents();
 
 			if (this->m_Cancelled) {
-				this->m_Archive.reset();
+				for (auto& archive : this->m_Archives) {
+					archive.reset();
+				}
 				return 0;
 			}
 
-			const QString& filepath = iterator.next();
+			const QFileInfo& fileInfo = iterator.nextFileInfo();
+			const QString& filepath = fileInfo.absoluteFilePath();
 			const bool ignored = this->m_ArchiveBuilderHelper->isFileIgnorable(filepath.toStdWString(), rootDirFiles);
 
 			Q_EMIT this->valueChanged(++count);
 			if (ignored || !filepath.endsWith(".dds", Qt::CaseInsensitive)) {
 				continue;
 			}
 
+			currentSize += fileInfo.size();
+			if (currentSize > SIZE_LIMIT) {
+				currentSize = fileInfo.size();
+				this->m_Archives.back()->set_compressed(!static_cast<bool>(incompressibleFiles));
+				this->m_Archives.back()->set_dds_callback(TextureArchiveBuilder::DDSCallback, this->getRootPath().toStdWString());
+				incompressibleFiles = 0;
+				compressibleFiles = 0;
+				this->m_Archives.emplace_back(std::make_unique<libbsarch::bs_archive_auto>(this->m_ArchiveType));
+				this->setShareData(true);
+			}
+
 			this->m_ArchiveBuilderHelper->isIncompressible(filepath.toStdWString()) ? ++incompressibleFiles : ++compressibleFiles;
 			auto fileBlob = disk_blob(
 				 dirString,
 				 filepath.toStdWString());
-			this->m_Archive->add_file_from_disk(fileBlob);
+			this->m_Archives.back()->add_file_from_disk(fileBlob);
 			qDebug() << "file is: " << filepath;
 		}
-		this->m_Archive->set_compressed(!static_cast<bool>(incompressibleFiles));
-		this->m_Archive->set_dds_callback(TextureArchiveBuilder::DDSCallback, this->getRootPath().toStdWString());
+		this->m_Archives.back()->set_compressed(!static_cast<bool>(incompressibleFiles));
+		this->m_Archives.back()->set_dds_callback(TextureArchiveBuilder::DDSCallback, this->getRootPath().toStdWString());
 		return incompressibleFiles + compressibleFiles;
 	}
 
 	void TextureArchiveBuilder::setShareData(const bool value)
 	{
-		this->m_Archive->set_share_data(value);
+		this->m_Archives.back()->set_share_data(value);
 	}
 
-	std::unique_ptr<libbsarch::bs_archive_auto> TextureArchiveBuilder::getArchive()
+	std::vector<std::unique_ptr<libbsarch::bs_archive_auto>> TextureArchiveBuilder::getArchives()
 	{
-		return std::move(this->m_Archive);
+		return std::move(this->m_Archives);
 	}
 
 	uint32_t TextureArchiveBuilder::getFileCount() const