Download files from direct URLs via a command line parameter (#1873)

* Download files via a command line parameter
* Use a GET parameter to predetermine Content-Disposition header in S3-compatible temporary URLs

src/commandline.cpp

@@ -2,6 +2,7 @@
 #include "env.h"
 #include "instancemanager.h"
 #include "loglist.h"
+#include "messagedialog.h"
 #include "multiprocess.h"
 #include "organizercore.h"
 #include "shared/appconfig.h"
@@ -49,8 +50,8 @@ CommandLine::CommandLine() : m_command(nullptr)
 {
   createOptions();
 
-  add<RunCommand, ReloadPluginCommand, RefreshCommand, CrashDumpCommand,
-      LaunchCommand>();
+  add<RunCommand, ReloadPluginCommand, DownloadFileCommand, RefreshCommand,
+      CrashDumpCommand, LaunchCommand>();
 }
 
 std::optional<int> CommandLine::process(const std::wstring& line)
@@ -833,6 +834,51 @@ std::optional<int> ReloadPluginCommand::runPostOrganizer(OrganizerCore& core)
   return {};
 }
 
+Command::Meta DownloadFileCommand::meta() const
+{
+  return {"download", "downloads a file", "URL", ""};
+}
+
+po::options_description DownloadFileCommand::getInternalOptions() const
+{
+  po::options_description d;
+
+  d.add_options()("URL", po::value<std::string>()->required(), "file URL");
+
+  return d;
+}
+
+po::positional_options_description DownloadFileCommand::getPositional() const
+{
+  po::positional_options_description d;
+
+  d.add("URL", 1);
+
+  return d;
+}
+
+bool DownloadFileCommand::canForwardToPrimary() const
+{
+  return true;
+}
+
+std::optional<int> DownloadFileCommand::runPostOrganizer(OrganizerCore& core)
+{
+  const QString url = QString::fromStdString(vm()["URL"].as<std::string>());
+
+  if (!url.startsWith("https://")) {
+    reportError(QObject::tr("Download URL must start with https://"));
+    return 1;
+  }
+
+  log::debug("starting direct download from command line: {}", url.toStdString());
+  MessageDialog::showMessage(QObject::tr("Download started"), qApp->activeWindow(),
+                             false);
+  core.downloadManager()->startDownloadURLs(QStringList() << url);
+
+  return {};
+}
+
 Command::Meta RefreshCommand::meta() const
 {
   return {"refresh", "refreshes MO (same as F5)", "", ""};

src/commandline.h

@@ -205,6 +205,20 @@ class ReloadPluginCommand : public Command
   std::optional<int> runPostOrganizer(OrganizerCore& core) override;
 };
 
+// downloads a file
+//
+class DownloadFileCommand : public Command
+{
+protected:
+  Meta meta() const override;
+
+  po::options_description getInternalOptions() const override;
+  po::positional_options_description getPositional() const override;
+
+  bool canForwardToPrimary() const override;
+  std::optional<int> runPostOrganizer(OrganizerCore& core) override;
+};
+
 // refreshes mo
 //
 class RefreshCommand : public Command

src/downloadmanager.cpp

@@ -435,6 +435,23 @@ bool DownloadManager::addDownload(const QStringList& URLs, QString gameName, int
   QString fileName = QFileInfo(URLs.first()).fileName();
   if (fileName.isEmpty()) {
     fileName = "unknown";
+  } else {
+    fileName = QUrl::fromPercentEncoding(fileName.toUtf8());
+  }
+
+  // Temporary URLs for S3-compatible storage are signed for a single method, removing
+  // the ability to make HEAD requests to such URLs. We can use the
+  // response-content-disposition GET parameter, setting the Content-Disposition header,
+  // to predetermine intended file name without a subrequest.
+  if (fileName.contains("response-content-disposition=")) {
+    std::regex exp("filename=\"(.+)\"");
+    std::cmatch result;
+    if (std::regex_search(fileName.toStdString().c_str(), result, exp)) {
+      fileName = MOBase::sanitizeFileName(QString::fromUtf8(result.str(1).c_str()));
+      if (fileName.isEmpty()) {
+        fileName = "unknown";
+      }
+    }
   }
 
   QUrl preferredUrl = QUrl::fromEncoded(URLs.first().toLocal8Bit());
@@ -446,6 +463,9 @@ bool DownloadManager::addDownload(const QStringList& URLs, QString gameName, int
   QNetworkRequest request(preferredUrl);
   request.setHeader(QNetworkRequest::UserAgentHeader,
                     m_NexusInterface->getAccessManager()->userAgent());
+  request.setAttribute(QNetworkRequest::CacheSaveControlAttribute, false);
+  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
+                       QNetworkRequest::AlwaysNetwork);
   request.setHttp2Configuration(h2Conf);
   return addDownload(m_NexusInterface->getAccessManager()->get(request), URLs, fileName,
                      gameName, modID, fileID, fileInfo);
@@ -597,7 +617,10 @@ void DownloadManager::startDownload(QNetworkReply* reply, DownloadInfo* newDownl
     if (newDownload->m_State != STATE_DOWNLOADING &&
         newDownload->m_State != STATE_READY &&
         newDownload->m_State != STATE_FETCHINGMODINFO && reply->isFinished()) {
-      downloadFinished(indexByInfo(newDownload));
+      int index = indexByInfo(newDownload);
+      if (index >= 0) {
+        downloadFinished(index);
+      }
       return;
     }
   } else
@@ -945,8 +968,7 @@ void DownloadManager::resumeDownloadInt(int index)
 
   // Check for finished download;
   if (info->m_TotalSize <= info->m_Output.size() && info->m_Reply != nullptr &&
-      info->m_Reply->isOpen() && info->m_Reply->isFinished() &&
-      info->m_State != STATE_ERROR) {
+      info->m_Reply->isFinished() && info->m_State != STATE_ERROR) {
     setState(info, STATE_DOWNLOADING);
     downloadFinished(index);
     return;
@@ -1508,7 +1530,7 @@ QString DownloadManager::getDownloadFileName(const QString& baseName, bool renam
 QString DownloadManager::getFileNameFromNetworkReply(QNetworkReply* reply)
 {
   if (reply->hasRawHeader("Content-Disposition")) {
-    std::regex exp("filename=\"(.*)\"");
+    std::regex exp("filename=\"(.+)\"");
 
     std::cmatch result;
     if (std::regex_search(reply->rawHeader("Content-Disposition").constData(), result,
@@ -2142,10 +2164,14 @@ void DownloadManager::nxmRequestFailed(QString gameName, int modID, int fileID,
 void DownloadManager::downloadFinished(int index)
 {
   DownloadInfo* info;
-  if (index)
+  if (index > 0)
     info = m_ActiveDownloads[index];
-  else
+  else {
     info = findDownload(this->sender(), &index);
+    if (info == nullptr && index == 0) {
+      info = m_ActiveDownloads[index];
+    }
+  }
 
   if (info != nullptr) {
     QNetworkReply* reply = info->m_Reply;