From 59b5b1cfca1614df681055d98d602692f482f0e2 Mon Sep 17 00:00:00 2001 From: Eddoursul <89222451+eddoursul@users.noreply.github.com> Date: Mon, 18 Sep 2023 19:48:13 +0200 Subject: [PATCH] Download files from direct URLs via a command line parameter (#1873) * Download files via a command line parameter * Use a GET parameter to predetermine Content-Disposition header in S3-compatible temporary URLs --- src/commandline.cpp | 50 +++++++++++++++++++++++++++++++++++++++-- src/commandline.h | 14 ++++++++++++ src/downloadmanager.cpp | 38 ++++++++++++++++++++++++++----- 3 files changed, 94 insertions(+), 8 deletions(-) diff --git a/src/commandline.cpp b/src/commandline.cpp index 9082d710a..1d44e0435 100644 --- a/src/commandline.cpp +++ b/src/commandline.cpp @@ -2,6 +2,7 @@ #include "env.h" #include "instancemanager.h" #include "loglist.h" +#include "messagedialog.h" #include "multiprocess.h" #include "organizercore.h" #include "shared/appconfig.h" @@ -49,8 +50,8 @@ CommandLine::CommandLine() : m_command(nullptr) { createOptions(); - add(); + add(); } std::optional CommandLine::process(const std::wstring& line) @@ -833,6 +834,51 @@ std::optional ReloadPluginCommand::runPostOrganizer(OrganizerCore& core) return {}; } +Command::Meta DownloadFileCommand::meta() const +{ + return {"download", "downloads a file", "URL", ""}; +} + +po::options_description DownloadFileCommand::getInternalOptions() const +{ + po::options_description d; + + d.add_options()("URL", po::value()->required(), "file URL"); + + return d; +} + +po::positional_options_description DownloadFileCommand::getPositional() const +{ + po::positional_options_description d; + + d.add("URL", 1); + + return d; +} + +bool DownloadFileCommand::canForwardToPrimary() const +{ + return true; +} + +std::optional DownloadFileCommand::runPostOrganizer(OrganizerCore& core) +{ + const QString url = QString::fromStdString(vm()["URL"].as()); + + if (!url.startsWith("https://")) { + reportError(QObject::tr("Download URL must start with https://")); + return 1; + } + + log::debug("starting direct download from command line: {}", url.toStdString()); + MessageDialog::showMessage(QObject::tr("Download started"), qApp->activeWindow(), + false); + core.downloadManager()->startDownloadURLs(QStringList() << url); + + return {}; +} + Command::Meta RefreshCommand::meta() const { return {"refresh", "refreshes MO (same as F5)", "", ""}; diff --git a/src/commandline.h b/src/commandline.h index c93d77f5e..3d7821054 100644 --- a/src/commandline.h +++ b/src/commandline.h @@ -205,6 +205,20 @@ class ReloadPluginCommand : public Command std::optional runPostOrganizer(OrganizerCore& core) override; }; +// downloads a file +// +class DownloadFileCommand : public Command +{ +protected: + Meta meta() const override; + + po::options_description getInternalOptions() const override; + po::positional_options_description getPositional() const override; + + bool canForwardToPrimary() const override; + std::optional runPostOrganizer(OrganizerCore& core) override; +}; + // refreshes mo // class RefreshCommand : public Command diff --git a/src/downloadmanager.cpp b/src/downloadmanager.cpp index 6fb88c3b9..5ccbdb4da 100644 --- a/src/downloadmanager.cpp +++ b/src/downloadmanager.cpp @@ -435,6 +435,23 @@ bool DownloadManager::addDownload(const QStringList& URLs, QString gameName, int QString fileName = QFileInfo(URLs.first()).fileName(); if (fileName.isEmpty()) { fileName = "unknown"; + } else { + fileName = QUrl::fromPercentEncoding(fileName.toUtf8()); + } + + // Temporary URLs for S3-compatible storage are signed for a single method, removing + // the ability to make HEAD requests to such URLs. We can use the + // response-content-disposition GET parameter, setting the Content-Disposition header, + // to predetermine intended file name without a subrequest. + if (fileName.contains("response-content-disposition=")) { + std::regex exp("filename=\"(.+)\""); + std::cmatch result; + if (std::regex_search(fileName.toStdString().c_str(), result, exp)) { + fileName = MOBase::sanitizeFileName(QString::fromUtf8(result.str(1).c_str())); + if (fileName.isEmpty()) { + fileName = "unknown"; + } + } } QUrl preferredUrl = QUrl::fromEncoded(URLs.first().toLocal8Bit()); @@ -446,6 +463,9 @@ bool DownloadManager::addDownload(const QStringList& URLs, QString gameName, int QNetworkRequest request(preferredUrl); request.setHeader(QNetworkRequest::UserAgentHeader, m_NexusInterface->getAccessManager()->userAgent()); + request.setAttribute(QNetworkRequest::CacheSaveControlAttribute, false); + request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, + QNetworkRequest::AlwaysNetwork); request.setHttp2Configuration(h2Conf); return addDownload(m_NexusInterface->getAccessManager()->get(request), URLs, fileName, gameName, modID, fileID, fileInfo); @@ -597,7 +617,10 @@ void DownloadManager::startDownload(QNetworkReply* reply, DownloadInfo* newDownl if (newDownload->m_State != STATE_DOWNLOADING && newDownload->m_State != STATE_READY && newDownload->m_State != STATE_FETCHINGMODINFO && reply->isFinished()) { - downloadFinished(indexByInfo(newDownload)); + int index = indexByInfo(newDownload); + if (index >= 0) { + downloadFinished(index); + } return; } } else @@ -945,8 +968,7 @@ void DownloadManager::resumeDownloadInt(int index) // Check for finished download; if (info->m_TotalSize <= info->m_Output.size() && info->m_Reply != nullptr && - info->m_Reply->isOpen() && info->m_Reply->isFinished() && - info->m_State != STATE_ERROR) { + info->m_Reply->isFinished() && info->m_State != STATE_ERROR) { setState(info, STATE_DOWNLOADING); downloadFinished(index); return; @@ -1508,7 +1530,7 @@ QString DownloadManager::getDownloadFileName(const QString& baseName, bool renam QString DownloadManager::getFileNameFromNetworkReply(QNetworkReply* reply) { if (reply->hasRawHeader("Content-Disposition")) { - std::regex exp("filename=\"(.*)\""); + std::regex exp("filename=\"(.+)\""); std::cmatch result; if (std::regex_search(reply->rawHeader("Content-Disposition").constData(), result, @@ -2142,10 +2164,14 @@ void DownloadManager::nxmRequestFailed(QString gameName, int modID, int fileID, void DownloadManager::downloadFinished(int index) { DownloadInfo* info; - if (index) + if (index > 0) info = m_ActiveDownloads[index]; - else + else { info = findDownload(this->sender(), &index); + if (info == nullptr && index == 0) { + info = m_ActiveDownloads[index]; + } + } if (info != nullptr) { QNetworkReply* reply = info->m_Reply;