Raise ratelimit user friendly and other network errors. #1109

cve_bin_tool/cvedb.py

@@ -28,6 +28,7 @@
     ErrorHandler,
     ErrorMode,
     SHAMismatch,
+    NVDRateLimit,
 )
 from cve_bin_tool.log import LOGGER
 from cve_bin_tool.version import check_latest_version
@@ -78,6 +79,7 @@ def __init__(
 
     async def getmeta(self, session, meta_url):
         async with session.get(meta_url) as response:
+            response.raise_for_status()
             return (
                 meta_url.replace(".meta", ".json.gz"),
                 dict(
@@ -91,6 +93,7 @@ async def getmeta(self, session, meta_url):
 
     async def nist_scrape(self, session):
         async with session.get(self.feed) as response:
+            response.raise_for_status()
             page = await response.text()
             json_meta_links = self.META_REGEX.findall(page)
             return dict(
@@ -135,6 +138,14 @@ async def cache_update(self, session, url, sha, chunk_size=16 * 1024):
         self.LOGGER.debug(f"Updating CVE cache for {filename}")
 
         async with session.get(url) as response:
+            # Raise better error message on ratelimit by NVD
+            if response.status == 403:
+                with ErrorHandler(mode=self.error_mode, logger=self.LOGGER):
+                    raise NVDRateLimit(
+                        f"{url} : download failed, you may have been rate limited."
+                    )
+            # Raise for all other 4xx errors
+            response.raise_for_status()
             gzip_data = await response.read()
         json_data = gzip.decompress(gzip_data)
         gotsha = hashlib.sha256(json_data).hexdigest().upper()
@@ -154,6 +165,7 @@ async def get_curl_versions(session):
         async with session.get(
             "https://curl.haxx.se/docs/vulnerabilities.html"
         ) as response:
+            response.raise_for_status()
             html = await response.text()
         matches = regex.finditer(html)
         return [match.group(1) for match in matches]
@@ -162,6 +174,7 @@ async def download_curl_version(self, session, version):
         async with session.get(
             f"https://curl.haxx.se/docs/vuln-{version}.html"
         ) as response:
+            response.raise_for_status()
             html = await response.text()
         soup = BeautifulSoup(html, "html.parser")
         table = soup.find("table")

cve_bin_tool/error_handler.py

@@ -64,6 +64,12 @@ class AttemptedToWriteOutsideCachedir(Exception):
     """
 
 
+class NVDRateLimit(Exception):
+    """
+    Raised if you have been ratelimited by NVD.
+    """
+
+
 class SHAMismatch(Exception):
     """
     Raised if the sha of a file in the cache was not what it should be.
@@ -160,4 +166,5 @@ def __exit__(self, exc_type, exc_val, exc_tb):
     UnknownConfigType: -14,
     CVEDataMissing: -15,
     InvalidCheckerError: -16,
+    NVDRateLimit: -17,
 }