Add results archiving

.gitignore

@@ -138,3 +138,4 @@ dmypy.json
 
 # Project
 logs/
+archives/

check.py

@@ -2,8 +2,9 @@
 import signal
 from time import sleep, time
 
-from utils import common, analyser, db
+from utils import analyser, common, db
 from utils import logger as main_logger
+from utils.common import archive_results
 from utils.logger import logger
 
 db_name: str = ''
@@ -62,6 +63,8 @@ def main() -> None:
     main()
     logger.info('Analysis pipeline done')
 
+    archive_results()
+
     end_time: float = time()
     execution_time: float = end_time - start_time
 

requirements.txt

@@ -13,7 +13,7 @@ pycodestyle==2.11.0
 pycparser==2.21
 pyOpenSSL==23.2.0
 requests==2.31.0
-soupsieve==2.4.1
+soupsieve==2.5
 tomli==2.0.1
 types-pyOpenSSL==23.2.0.2
 types-requests==2.31.0.2

utils/analyser.py

@@ -1,15 +1,15 @@
 import ssl
-from os import path, mkdir, remove, listdir
-from shutil import rmtree
+from os import mkdir, path
 from time import sleep
 from urllib.parse import quote, urlparse
 
 import requests
 from OpenSSL import crypto
 
 from utils import threading
+from utils.common import clean_logs_directory, clean_results_directory
+from utils.const import HEADER, SELF_SIGNED_CERTS, UNTRUSTED_CERTS
 from utils.logger import logger
-from utils.const import UNTRUSTED_CERTS, SELF_SIGNED_CERTS, HEADER
 
 
 def __get_root_cert(link: str):
@@ -100,15 +100,8 @@ def run_pipeline(link_batches: tuple, timeout: int) -> None:
     last_idx: int = len(link_batches)
     idx: int = 0
 
-    if path.exists('results'):
-        for content in listdir('results/'):
-            content_path = path.join('results/', content)
-            if path.isfile(content_path):
-                remove(content_path)
-            else:
-                rmtree(content_path, ignore_errors=True)
-    else:
-        mkdir('results')
+    clean_results_directory()
+    clean_logs_directory()
 
     mkdir('results/government')
     mkdir('results/social')

utils/common.py

@@ -1,3 +1,10 @@
+import datetime
+import os
+import zipfile
+from os import listdir, mkdir, path, remove
+from shutil import rmtree
+
+
 def read_links(filename: str) -> list[list[str]]:
     MAX_BATCH_SIZE: int = 8000
     with open(filename) as f:
@@ -31,3 +38,67 @@ def get_lines_count_in(file: str) -> int:
         exit(1)
 
     return count
+
+
+def archive_results(save_logs=False) -> None:
+    """Save results of the analysis to <date-time>.zip archive.
+
+    Keyword arguments:
+    save_logs -- add logs folder to archive (optional, default False)
+    """
+    # Get the current date and time
+    current_datetime = datetime.datetime.now()
+    date_time_str = current_datetime.strftime("%Y-%m-%d-%H-%M-%S")
+
+    # Define the archive filename
+    archive_filename = f"{date_time_str}.zip"
+
+    # Check if the 'results' folder exists
+    if not os.path.exists('results'):
+        print("Error: 'results' folder not found.")
+        return
+
+    # Create a new ZIP archive
+    with zipfile.ZipFile(archive_filename, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        # Add files from the 'results' folder to the archive
+        for root, _, files in os.walk('results'):
+            for file in files:
+                file_path = os.path.join(root, file)
+                zipf.write(file_path, os.path.relpath(file_path, 'results'))
+
+        # Add files from the 'logs' folder to the archive if save_logs is True
+        if save_logs and os.path.exists('logs'):
+            for root, _, files in os.walk('logs'):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    zipf.write(file_path, os.path.relpath(file_path, 'logs'))
+
+    # Move the created archive to the 'results' folder
+    if not path.exists('archives'):
+        mkdir('archives')
+    os.rename(archive_filename, os.path.join('archives', archive_filename))
+
+    print(
+        f"Results successfully archived to {os.path.join('results', archive_filename)}")
+
+
+def clean_logs_directory():
+    if path.exists('logs'):
+        for content in os.listdir('logs'):
+            content_path = path.join('logs', content)
+            if path.isfile(content_path):
+                remove(content_path)
+            else:
+                rmtree(content_path, ignore_errors=True)
+
+
+def clean_results_directory():
+    if path.exists('results'):
+        for content in listdir('results/'):
+            content_path = path.join('results/', content)
+            if path.isfile(content_path):
+                remove(content_path)
+            else:
+                rmtree(content_path, ignore_errors=True)
+    else:
+        mkdir('results')