Cache GameList API query in rahasher.py

docs/configs/config.rst

@@ -56,6 +56,8 @@ Syntax: ::
     rahasher:                           # RAHasher specific options
       username: "user"                  # RA username
       api_key: "1234567890abcde"        # RA API key
+      cache_period: 30                  # Cache period for GameList in days. Since this is a heavy API operation, RA
+                                        # suggests caching this aggressively. Defaults to 30
 
     dupeparser:                         # DupeParser specific options
       use_dat: true                     # OPTIONAL. Whether to use .dat files or not. Defaults to true

docs/modules/rahasher.rst

@@ -6,7 +6,9 @@ The RAHasher will download a list of compatible ROM hashes from RetroAchievement
 achievements.
 
 This requires API access, so you need to supply your RA username and API key. For how to do this, see
-`this page <https://api-docs.retroachievements.org/#api-access>`_.
+`this page <https://api-docs.retroachievements.org/#api-access>`_. Because GetGameList is heavy on the API,
+there is a cache implemented that by default will only query the full game list once a month. If you need
+to change this, you can with ``cache_period``, but do so at your own risk!
 
 For more details on the RAHasher arguments, see the :doc:`config file documentation <../configs/config>`.
 

romsearch/configs/sample_config.yml

@@ -34,6 +34,11 @@ romdownloader:
   remote_name: 'rclone_remote'
   sync_all: false
 
+rahasher:
+  username: "user"
+  api_key: "1234567890abcde"
+  cache_period: 30
+
 dupeparser:
   use_dat: true
   use_retool: true

romsearch/gui/gui_config.py

@@ -163,6 +163,7 @@ def __init__(self,
         self.rahasher_text_fields = {
             "username": self.ui.lineEditConfigRAHasherUsername,
             "api_key": self.ui.lineEditConfigRAHasherAPIKey,
+            "cache_period": self.ui.lineEditConfigRAHasherCachePeriod,
         }
 
         self.discord_text_fields = {

romsearch/gui/layout_romsearch.py

@@ -756,6 +756,18 @@ def setupUi(self, RomSearch):
 
         self.verticalLayoutConfigRAHasherMiddle.addWidget(self.lineEditConfigRAHasherAPIKey)
 
+        self.labelConfigRAHasherCachePeriodDescription = QLabel(self.tabConfigRAHasher)
+        self.labelConfigRAHasherCachePeriodDescription.setObjectName(u"labelConfigRAHasherCachePeriodDescription")
+        self.labelConfigRAHasherCachePeriodDescription.setWordWrap(True)
+
+        self.verticalLayoutConfigRAHasherMiddle.addWidget(self.labelConfigRAHasherCachePeriodDescription)
+
+        self.lineEditConfigRAHasherCachePeriod = QLineEdit(self.tabConfigRAHasher)
+        self.lineEditConfigRAHasherCachePeriod.setObjectName(u"lineEditConfigRAHasherCachePeriod")
+        self.lineEditConfigRAHasherCachePeriod.setFrame(True)
+
+        self.verticalLayoutConfigRAHasherMiddle.addWidget(self.lineEditConfigRAHasherCachePeriod)
+
         self.lineConfigRAHasherDividerBottom = QFrame(self.tabConfigRAHasher)
         self.lineConfigRAHasherDividerBottom.setObjectName(u"lineConfigRAHasherDividerBottom")
         self.lineConfigRAHasherDividerBottom.setFrameShadow(QFrame.Shadow.Plain)
@@ -1623,6 +1635,9 @@ def retranslateUi(self, RomSearch):
         self.labelConfigRAHasherAPIKeyDescription.setText(QCoreApplication.translate("RomSearch", u"RetroAchievements API Key", None))
         self.lineEditConfigRAHasherAPIKey.setText("")
         self.lineEditConfigRAHasherAPIKey.setPlaceholderText(QCoreApplication.translate("RomSearch", u"1234567890abcde", None))
+        self.labelConfigRAHasherCachePeriodDescription.setText(QCoreApplication.translate("RomSearch", u"Cache Period", None))
+        self.lineEditConfigRAHasherCachePeriod.setText(QCoreApplication.translate("RomSearch", u"30", None))
+        self.lineEditConfigRAHasherCachePeriod.setPlaceholderText(QCoreApplication.translate("RomSearch", u"30", None))
         self.tabWidgetConfig.setTabText(self.tabWidgetConfig.indexOf(self.tabConfigRAHasher), QCoreApplication.translate("RomSearch", u"RAHasher", None))
 #if QT_CONFIG(statustip)
         self.checkBoxConfigDupeParserUseDat.setStatusTip(QCoreApplication.translate("RomSearch", u"Whether to use the dat file to figure out dupes. Default checked", None))

romsearch/gui/layout_romsearch.ui

@@ -1270,6 +1270,29 @@
                   </property>
                  </widget>
                 </item>
+                <item>
+                 <widget class="QLabel" name="labelConfigRAHasherCachePeriodDescription">
+                  <property name="text">
+                   <string>Cache Period</string>
+                  </property>
+                  <property name="wordWrap">
+                   <bool>true</bool>
+                  </property>
+                 </widget>
+                </item>
+                <item>
+                 <widget class="QLineEdit" name="lineEditConfigRAHasherCachePeriod">
+                  <property name="text">
+                   <string>30</string>
+                  </property>
+                  <property name="frame">
+                   <bool>true</bool>
+                  </property>
+                  <property name="placeholderText">
+                   <string>30</string>
+                  </property>
+                 </widget>
+                </item>
                 <item>
                  <widget class="Line" name="lineConfigRAHasherDividerBottom">
                   <property name="frameShadow">
@@ -2527,7 +2550,7 @@
  <resources/>
  <connections/>
  <buttongroups>
-  <buttongroup name="radioButtonConfigRomsearchMethod"/>
   <buttongroup name="radioButtonConfigLoggerLevel"/>
+  <buttongroup name="radioButtonConfigRomsearchMethod"/>
  </buttongroups>
 </ui>

romsearch/modules/rahasher.py

@@ -1,16 +1,40 @@
 import copy
 import os
 import requests
+from datetime import datetime
 
 import romsearch
 from ..util import (centred_string,
                     load_yml,
                     setup_logger,
+                    load_json,
                     save_json,
+                    get_file_time,
                     )
 
 RA_URL = "https://retroachievements.org/API"
 
+
+def format_game_list(in_file,
+                     ):
+    """Format the GameList neatly
+
+    Args:
+        in_file (str): Input GameList file
+    """
+
+    data = load_json(in_file)
+
+    ra_game_info = {}
+    for d in data:
+        # RA uses "Title: Subtitle" logic, while No-Intro/Redump do not
+        clean_title = d['Title'].replace(":", " -")
+
+        ra_game_info[clean_title] = copy.deepcopy(d)
+
+    return ra_game_info
+
+
 class RAHasher:
 
     def __init__(self,
@@ -22,7 +46,7 @@ def __init__(self,
                  log_line_sep="=",
                  log_line_length=100,
                  ):
-        """Get supported ROM hashes for RetroAchievements
+        """Get supported ROM files and hashes for RetroAchievements
 
         This works per-platform, so must be specified here
 
@@ -68,14 +92,19 @@ def __init__(self,
         self.username = self.config.get("rahasher", {}).get("username", None)
         self.api_key = self.config.get("rahasher", {}).get("api_key", None)
 
+        cache_period = self.config.get("rahasher", {}).get("cache_period", 30)
+        if isinstance(cache_period, str):
+            cache_period = float(cache_period)
+        self.cache_period = cache_period
+
         self.log_line_sep = log_line_sep
         self.log_line_length = log_line_length
 
     def run(self):
         """Run the RA hasher"""
 
         run_rahasher = True
-        ra_hashes = None
+        ra_game_info = None
 
         if self.ra_hash_dir is None:
             self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
@@ -114,9 +143,9 @@ def run(self):
             run_rahasher = False
 
         if run_rahasher:
-            ra_hashes = self.run_rahasher()
+            ra_game_info = self.run_rahasher()
 
-        return ra_hashes
+        return ra_game_info
 
     def run_rahasher(self):
         """The main meat of running the RA hasher"""
@@ -127,44 +156,72 @@ def run_rahasher(self):
                          )
         self.logger.info(f"{self.log_line_sep * self.log_line_length}")
 
-        ra_hashes = self.get_ra_hashes()
-        self.save_ra_hashes(ra_hashes)
+        ra_game_info = self.get_ra_game_info()
+        self.save_ra_game_info(ra_game_info)
 
         self.logger.info(f"{self.log_line_sep * self.log_line_length}")
 
-        return ra_hashes
+        return ra_game_info
+
+    def get_ra_game_info(self):
+        """Download or read in RA game info."""
+
+        game_list_file = os.path.join(self.ra_hash_dir, f"{self.platform} GameList.json")
+        game_list_modtime = get_file_time(game_list_file,
+                                          return_as_str=False,
+                                          )
+
+        cur_time = datetime.now()
+        diff = cur_time - game_list_modtime
 
-    def get_ra_hashes(self):
-        """Download the RA hashes using the API. Parse to JSON"""
+        if diff.days > self.cache_period:
+            self.logger.info(centred_string(f"Downloading latest GameList to {game_list_file}",
+                                            total_length=self.log_line_length)
+                             )
+            self.download_ra_game_list(out_file=game_list_file)
+
+        ra_game_info = format_game_list(game_list_file)
+
+        return ra_game_info
+
+    def download_ra_game_list(self,
+                              out_file,
+                              ):
+        """Download the RA GameList using the API
+
+        Args:
+            out_file (string): output file name.
+        """
 
         console_id = self.platform_config["ra_id"]
 
         url = f"{RA_URL}/API_GetGameList.php?z={self.username}&y={self.api_key}&i={console_id}&h=1&f=1"
         resp = requests.get(url=url)
         data = resp.json()
 
-        hash_dict = {}
-        for d in data:
+        save_json(data, out_file)
 
-            # RA uses "Title: Subtitle" logic, while No-Intro/Redump do not
-            clean_title = d['Title'].replace(":", " -")
-
-            hash_dict[clean_title] = copy.deepcopy(d)
-
-        return hash_dict
+        return True
 
-    def save_ra_hashes(self, ra_hashes):
-        """Save out the RA hashes to a file"""
+    def save_ra_game_info(self,
+                          ra_game_info,
+                          ):
+        """Save out the RA game info to a file"""
 
         if not os.path.exists(self.ra_hash_dir):
             os.makedirs(self.ra_hash_dir)
 
-        out_name = os.path.join(self.ra_hash_dir, f"{self.platform}.json")
+        out_name = os.path.join(self.ra_hash_dir,
+                                f"{self.platform}.json",
+                                )
 
-        self.logger.info(centred_string(f"Saving to {out_name}",
-                                        total_length=self.log_line_length)
+        self.logger.info(centred_string(f"Saving full game info to {out_name}",
+                                        total_length=self.log_line_length
+                                        )
                          )
 
-        save_json(ra_hashes, out_name)
+        save_json(ra_game_info,
+                  out_name,
+                  )
 
         return True

romsearch/util/general.py

@@ -60,15 +60,25 @@ def get_parent_name(game_name,
 
 
 def get_file_time(f,
-                  datetime_format,
+                  datetime_format="%Y/%m/%d, %H:%M:%S",
+                  return_as_str=True
                   ):
-    """Get created file time from the file itself"""
+    """Get created file time from the file itself
+
+    Args:
+        f (str): Filename
+        datetime_format (str, optional): Date and time format. Defaults to "%Y/%m/%d %H:%M:%S"
+        return_as_str (bool, optional): Return string or full datetime. Defaults to True
+    """
 
     if os.path.exists(f):
         ti_m = os.path.getmtime(f)
         date_ti_m = datetime.strptime(time.ctime(ti_m), "%a %b %d %H:%M:%S %Y")
     else:
         date_ti_m = datetime(year=1900, month=1, day=1, hour=0, minute=0, second=0)
-    date_ti_m_str = date_ti_m.strftime(format=datetime_format)
 
-    return date_ti_m_str
+    if return_as_str:
+        date_ti_m_str = date_ti_m.strftime(format=datetime_format)
+        return date_ti_m_str
+    else:
+        return date_ti_m