Mass Metadata Parsing: Prevent re-querying and manual prompts to enter missing data (#2135)

* Add a button to "Query Info" of every download in the list

---------

Co-authored-by: Deewens <dudonadrien@gmail.com>
Co-authored-by: KenJyn76 <liambonilla@gmail.com>
Co-authored-by: Al <26797547+Al12rs@users.noreply.github.com>

Author: 4 people

src/downloadmanager.cpp

@@ -441,6 +441,36 @@ void DownloadManager::refreshList()
   }
 }
 
+void DownloadManager::queryDownloadListInfo()
+{
+  int incompleteCount = 0;
+  for (size_t i = 0; i < m_ActiveDownloads.size(); i++) {
+    if (isInfoIncomplete(i)) {
+      incompleteCount++;
+    }
+  }
+
+  if (incompleteCount <= 5 ||
+      QMessageBox::question(
+          m_ParentWidget, tr("Query Metadata"),
+          tr("There are %1 downloads with incomplete metadata.\n\n"
+             "Do you want to fetch all incomplete metadata?\n"
+             "API requests will be consumed, and Mod Organizer may stutter.")
+              .arg(incompleteCount),
+          QMessageBox::Yes | QMessageBox::No) == QMessageBox::Yes) {
+    TimeThis tt("DownloadManager::queryDownloadListInfo()");
+    log::info("Querying metadata for every download with incomplete info...");
+    startDisableDirWatcher();
+    for (size_t i = 0; i < m_ActiveDownloads.size(); i++) {
+      if (isInfoIncomplete(i)) {
+        queryInfoMd5(i, false);
+      }
+    }
+    endDisableDirWatcher();
+    log::info("Metadata has been retrieved successfully!");
+  }
+}
+
 bool DownloadManager::addDownload(const QStringList& URLs, QString gameName, int modID,
                                   int fileID, const ModRepositoryFileInfo* fileInfo)
 {
@@ -1054,10 +1084,10 @@ void DownloadManager::queryInfo(int index)
     QString fileName = getFileName(index);
     QString ignore;
     NexusInterface::interpretNexusFileName(fileName, ignore, info->m_FileInfo->modID,
-                                           true);
+                                           info->m_AskIfNotFound);
     if (info->m_FileInfo->modID < 0) {
       bool ok   = false;
-      int modId = QInputDialog::getInt(nullptr, tr("Please enter the nexus mod id"),
+      int modId = QInputDialog::getInt(nullptr, tr("Please enter the Nexus mod ID"),
                                        tr("Mod ID:"), 1, 1,
                                        std::numeric_limits<int>::max(), 1, &ok);
       // careful now: while the dialog was displayed, events were processed.
@@ -1068,7 +1098,13 @@ void DownloadManager::queryInfo(int index)
     }
   }
 
-  if (info->m_FileInfo->gameName.size() == 0) {
+  if (info->m_FileInfo->gameName.isEmpty()) {
+    if (!info->m_AskIfNotFound) {
+      // in case the info couldn't be retrieved from Nexus due to connection issues,
+      // don't create a meta file so you can query again after the issues are resolved.
+      return;
+    }
+
     SelectionDialog selection(
         tr("Please select the source game code for %1").arg(getFileName(index)));
 
@@ -1091,7 +1127,7 @@ void DownloadManager::queryInfo(int index)
   setState(info, STATE_FETCHINGMODINFO);
 }
 
-void DownloadManager::queryInfoMd5(int index)
+void DownloadManager::queryInfoMd5(int index, bool askIfNotFound)
 {
   if ((index < 0) || (index >= m_ActiveDownloads.size())) {
     reportError(tr("query: invalid download index %1").arg(index));
@@ -1148,8 +1184,9 @@ void DownloadManager::queryInfoMd5(int index)
   progress.close();
   downloadFile.close();
 
-  info->m_Hash      = hash.result();
-  info->m_ReQueried = true;
+  info->m_Hash          = hash.result();
+  info->m_ReQueried     = true;
+  info->m_AskIfNotFound = askIfNotFound;
   setState(info, STATE_FETCHINGMODINFO_MD5);
 }
 
@@ -2133,6 +2170,11 @@ void DownloadManager::nxmRequestFailed(QString gameName, int modID, int fileID,
         info->m_GamesToQuery.pop_front();
         setState(info, STATE_FETCHINGMODINFO_MD5);
         return;
+      } else if (errorCode == 404 && !info->m_AskIfNotFound) {
+        // prevent re-querying (only possible with repository = "Nexus")
+        info->m_FileInfo->repository = "";
+        setState(info, STATE_READY);
+        return;
       } else {
         info->m_State = STATE_READY;
         queryInfo(index);

src/downloadmanager.h

@@ -108,6 +108,7 @@ class DownloadManager : public QObject
 
     int m_Tries;
     bool m_ReQueried;
+    bool m_AskIfNotFound;
 
     quint32 m_TaskProgressId;
 
@@ -143,7 +144,7 @@ class DownloadManager : public QObject
   private:
     DownloadInfo()
         : m_TotalSize(0), m_ReQueried(false), m_Hidden(false), m_HasData(false),
-          m_DownloadTimeLast(0), m_DownloadLast(0),
+          m_AskIfNotFound(true), m_DownloadTimeLast(0), m_DownloadLast(0),
           m_DownloadAcc(tag::rolling_window::window_size = 200),
           m_DownloadTimeAcc(tag::rolling_window::window_size = 200)
     {}
@@ -402,6 +403,11 @@ class DownloadManager : public QObject
    */
   void refreshList();
 
+  /**
+   * @brief Query infos for every download in the list
+   */
+  void queryDownloadListInfo();
+
 public:  // IDownloadManager interface:
   int startDownloadURLs(const QStringList& urls);
   int startDownloadNexusFile(const QString& gameName, int modID, int fileID);
@@ -493,7 +499,7 @@ public slots:
 
   void queryInfo(int index);
 
-  void queryInfoMd5(int index);
+  void queryInfoMd5(int index, bool askIfNotFound = true);
 
   void visitOnNexus(int index);
 

src/downloadstab.cpp

@@ -5,8 +5,9 @@
 #include "ui_mainwindow.h"
 
 DownloadsTab::DownloadsTab(OrganizerCore& core, Ui::MainWindow* mwui)
-    : m_core(core), ui{mwui->btnRefreshDownloads, mwui->downloadView,
-                       mwui->showHiddenBox, mwui->downloadFilterEdit}
+    : m_core(core),
+      ui{mwui->btnRefreshDownloads, mwui->btnQueryDownloadsInfo, mwui->downloadView,
+         mwui->showHiddenBox, mwui->downloadFilterEdit}
 {
   DownloadList* sourceModel = new DownloadList(m_core, ui.list);
 
@@ -26,6 +27,9 @@ DownloadsTab::DownloadsTab(OrganizerCore& core, Ui::MainWindow* mwui)
   connect(ui.refresh, &QPushButton::clicked, [&] {
     refresh();
   });
+  connect(ui.queryInfos, &QPushButton::clicked, [&] {
+    queryInfos();
+  });
   connect(ui.list, SIGNAL(installDownload(int)), &m_core, SLOT(installDownload(int)));
   connect(ui.list, SIGNAL(queryInfo(int)), m_core.downloadManager(),
           SLOT(queryInfo(int)));
@@ -80,6 +84,22 @@ void DownloadsTab::refresh()
   m_core.downloadManager()->refreshList();
 }
 
+void DownloadsTab::queryInfos()
+{
+  if (m_core.settings().network().offlineMode()) {
+    if (QMessageBox::warning(nullptr, tr("Query Metadata"),
+                             tr("Cannot query metadata while offline mode is enabled. "
+                                "Do you want to disable offline mode?"),
+                             QMessageBox::Yes | QMessageBox::No) == QMessageBox::Yes) {
+      m_core.settings().network().setOfflineMode(false);
+    } else {
+      return;
+    }
+  }
+
+  m_core.downloadManager()->queryDownloadListInfo();
+}
+
 void DownloadsTab::resumeDownload(int downloadIndex)
 {
   m_core.loggedInAction(ui.list, [this, downloadIndex] {

src/downloadstab.h

@@ -23,6 +23,7 @@ class DownloadsTab : public QObject
   struct DownloadsTabUi
   {
     QPushButton* refresh;
+    QPushButton* queryInfos;
     DownloadListView* list;
     QCheckBox* showHidden;
     QLineEdit* filter;
@@ -33,6 +34,12 @@ class DownloadsTab : public QObject
   MOBase::FilterWidget m_filter;
 
   void refresh();
+
+  /**
+   * @brief Handle click on the "Query infos" button
+   **/
+  void queryInfos();
+
   void resumeDownload(int downloadIndex);
 };
 

src/mainwindow.ui

@@ -1345,6 +1345,26 @@
                  </property>
                 </spacer>
                </item>
+               <item>
+                <widget class="QPushButton" name="btnQueryDownloadsInfo">
+                 <property name="sizePolicy">
+                  <sizepolicy hsizetype="Minimum" vsizetype="Fixed">
+                   <horstretch>0</horstretch>
+                   <verstretch>0</verstretch>
+                  </sizepolicy>
+                 </property>
+                 <property name="contextMenuPolicy">
+                  <enum>Qt::ContextMenuPolicy::DefaultContextMenu</enum>
+                 </property>
+                 <property name="text">
+                  <string>Query Metadata</string>
+                 </property>
+                 <property name="icon">
+                  <iconset resource="resources.qrc">
+                   <normaloff>:/MO/gui/resources/system-search.png</normaloff>:/MO/gui/resources/system-search.png</iconset>
+                 </property>
+                </widget>
+               </item>
               </layout>
              </item>
              <item>