Data Aggregator Rewrite

.gitignore

@@ -94,3 +94,5 @@ openwpm/Extension/firefox/dist
 openwpm/Extension/firefox/openwpm.xpi
 openwpm/Extension/firefox/src/content.js
 openwpm/Extension/firefox/src/feature.js
+
+datadir

.pre-commit-config.yaml

@@ -4,7 +4,16 @@ repos:
     hooks:
     -   id: isort
   - repo: https://github.com/psf/black
-    rev: 20.8b1 # Replace by any tag/version: https://github.com/psf/black/tags
+    rev: 19.10b0
     hooks:
       - id: black
-        language_version: python3 # Should be a command that runs python3.6+
+        language_version: python3
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.790
+    hooks:
+      - id: mypy
+        additional_dependencies: [pytest]
+        # We may need to add more and more dependencies here, as pre-commit
+        # runs in an environment without our dependencies
+
+

.travis.yml

@@ -6,11 +6,11 @@ env:
 # Once we add and remove tests, this distribution may become unbalanced.
 # Feel free to move tests around to make the running time of the jobs
 # as close as possible.
-  - TESTS=test_[a-e]*
-  - TESTS=test_[f-h]*
-  - TESTS=test_[i-r,t-z]*
+  - TESTS=test/test_[a-e]*
+  - TESTS=test/test_[f-h]*
+  - TESTS=test/test_[i-r,t-z]*
 # test_simple_commands.py is slow due to parametrization.
-  - TESTS=test_[s]*
+  - TESTS=test/test_[s]*
   - TESTS=webextension
 git:
   depth: 3
@@ -36,15 +36,11 @@ after_success:
 
 jobs:
   include:
-    - language:
-      python:
+    - language: "python"
       env:
         - TESTS="Docker"
       services:
         - docker
-      before_install:
-      before_script:
-      install:
       script:
         - docker build -f Dockerfile -t openwpm .
         - ./scripts/deploy-to-dockerhub.sh

crawler.py

@@ -4,42 +4,53 @@
 import signal
 import sys
 import time
+from pathlib import Path
 from threading import Lock
 from typing import Any, Callable, List
 
-import boto3
 import sentry_sdk
 
+from openwpm import mp_logger
 from openwpm.command_sequence import CommandSequence
 from openwpm.config import BrowserParams, ManagerParams
-from openwpm.mp_logger import parse_config_from_env
-from openwpm.task_manager import TaskManager, load_default_params
+from openwpm.storage.cloud_storage.gcp_storage import (
+    GcsStructuredProvider,
+    GcsUnstructuredProvider,
+)
+from openwpm.task_manager import TaskManager
 from openwpm.utilities import rediswq
-from test.utilities import LocalS3Session, local_s3_bucket
 
 # Configuration via environment variables
+# Crawler specific config
 REDIS_HOST = os.getenv("REDIS_HOST", "redis-box")
 REDIS_QUEUE_NAME = os.getenv("REDIS_QUEUE_NAME", "crawl-queue")
+MAX_JOB_RETRIES = int(os.getenv("MAX_JOB_RETRIES", "2"))
+DWELL_TIME = int(os.getenv("DWELL_TIME", "10"))
+TIMEOUT = int(os.getenv("TIMEOUT", "60"))
+
+# Storage Provider Params
 CRAWL_DIRECTORY = os.getenv("CRAWL_DIRECTORY", "crawl-data")
-S3_BUCKET = os.getenv("S3_BUCKET", "openwpm-crawls")
+GCS_BUCKET = os.getenv("GCS_BUCKET", "openwpm-crawls")
+GCP_PROJECT = os.getenv("GCP_PROJECT", "senglehardt-openwpm-test-1")
+AUTH_TOKEN = os.getenv("GCP_AUTH_TOKEN", "cloud")
+
+# Browser Params
 DISPLAY_MODE = os.getenv("DISPLAY_MODE", "headless")
 HTTP_INSTRUMENT = os.getenv("HTTP_INSTRUMENT", "1") == "1"
 COOKIE_INSTRUMENT = os.getenv("COOKIE_INSTRUMENT", "1") == "1"
 NAVIGATION_INSTRUMENT = os.getenv("NAVIGATION_INSTRUMENT", "1") == "1"
 JS_INSTRUMENT = os.getenv("JS_INSTRUMENT", "1") == "1"
 CALLSTACK_INSTRUMENT = os.getenv("CALLSTACK_INSTRUMENT", "1") == "1"
-JS_INSTRUMENT_SETTINGS = os.getenv(
-    "JS_INSTRUMENT_SETTINGS", '["collection_fingerprinting"]'
+JS_INSTRUMENT_SETTINGS = json.loads(
+    os.getenv("JS_INSTRUMENT_SETTINGS", '["collection_fingerprinting"]')
 )
+
 SAVE_CONTENT = os.getenv("SAVE_CONTENT", "")
 PREFS = os.getenv("PREFS", None)
-DWELL_TIME = int(os.getenv("DWELL_TIME", "10"))
-TIMEOUT = int(os.getenv("TIMEOUT", "60"))
-SENTRY_DSN = os.getenv("SENTRY_DSN", None)
-LOGGER_SETTINGS = parse_config_from_env()
-MAX_JOB_RETRIES = int(os.getenv("MAX_JOB_RETRIES", "2"))
 
-JS_INSTRUMENT_SETTINGS = json.loads(JS_INSTRUMENT_SETTINGS)
+
+SENTRY_DSN = os.getenv("SENTRY_DSN", None)
+LOGGER_SETTINGS = mp_logger.parse_config_from_env()
 
 if CALLSTACK_INSTRUMENT is True:
     # Must have JS_INSTRUMENT True for CALLSTACK_INSTRUMENT to work
@@ -74,29 +85,41 @@
         browser_params[i].prefs = json.loads(PREFS)
 
 # Manager configuration
-manager_params.data_directory = "~/Desktop/%s/" % CRAWL_DIRECTORY
-manager_params.log_directory = "~/Desktop/%s/" % CRAWL_DIRECTORY
+manager_params.data_directory = Path("~/Desktop/") / CRAWL_DIRECTORY
+manager_params.log_directory = Path("~/Desktop/") / CRAWL_DIRECTORY
 manager_params.output_format = "s3"
-manager_params.s3_bucket = S3_BUCKET
+manager_params.s3_bucket = GCS_BUCKET
 manager_params.s3_directory = CRAWL_DIRECTORY
 
-# Allow the use of localstack's mock s3 service
-S3_ENDPOINT = os.getenv("S3_ENDPOINT")
-if S3_ENDPOINT:
-    boto3.DEFAULT_SESSION = LocalS3Session(endpoint_url=S3_ENDPOINT)
-    manager_params.s3_bucket = local_s3_bucket(boto3.resource("s3"), name=S3_BUCKET)
-
+structured = GcsStructuredProvider(
+    project=GCP_PROJECT,
+    bucket_name=GCS_BUCKET,
+    base_path=CRAWL_DIRECTORY,
+    token=AUTH_TOKEN,
+)
+unstructured = GcsUnstructuredProvider(
+    project=GCP_PROJECT,
+    bucket_name=GCS_BUCKET,
+    base_path=CRAWL_DIRECTORY,
+    token=AUTH_TOKEN,
+)
 # Instantiates the measurement platform
 # Commands time out by default after 60 seconds
-manager = TaskManager(manager_params, browser_params, logger_kwargs=LOGGER_SETTINGS)
+manager = TaskManager(
+    manager_params,
+    browser_params,
+    structured,
+    unstructured,
+    logger_kwargs=LOGGER_SETTINGS,
+)
 
 # At this point, Sentry should be initiated
 if SENTRY_DSN:
     # Add crawler.py-specific context
     with sentry_sdk.configure_scope() as scope:
         # tags generate breakdown charts and search filters
         scope.set_tag("CRAWL_DIRECTORY", CRAWL_DIRECTORY)
-        scope.set_tag("S3_BUCKET", S3_BUCKET)
+        scope.set_tag("S3_BUCKET", GCS_BUCKET)
         scope.set_tag("DISPLAY_MODE", DISPLAY_MODE)
         scope.set_tag("HTTP_INSTRUMENT", HTTP_INSTRUMENT)
         scope.set_tag("COOKIE_INSTRUMENT", COOKIE_INSTRUMENT)
@@ -108,14 +131,12 @@
         scope.set_tag("DWELL_TIME", DWELL_TIME)
         scope.set_tag("TIMEOUT", TIMEOUT)
         scope.set_tag("MAX_JOB_RETRIES", MAX_JOB_RETRIES)
-        scope.set_tag("CRAWL_REFERENCE", "%s/%s" % (S3_BUCKET, CRAWL_DIRECTORY))
+        scope.set_tag("CRAWL_REFERENCE", "%s/%s" % (GCS_BUCKET, CRAWL_DIRECTORY))
         # context adds addition information that may be of interest
-        scope.set_context("PREFS", PREFS)
+        if PREFS:
+            scope.set_context("PREFS", json.loads(PREFS))
         scope.set_context(
-            "crawl_config",
-            {
-                "REDIS_QUEUE_NAME": REDIS_QUEUE_NAME,
-            },
+            "crawl_config", {"REDIS_QUEUE_NAME": REDIS_QUEUE_NAME,},
         )
     # Send a sentry error message (temporarily - to easily be able
     # to compare error frequencies to crawl worker instance count)
@@ -159,9 +180,9 @@ def get_job_completion_callback(
     job_queue: rediswq.RedisWQ,
     job: bytes,
 ) -> Callable[[bool], None]:
-    def callback(sucess: bool) -> None:
+    def callback(success: bool) -> None:
         with unsaved_jobs_lock:
-            if sucess:
+            if success:
                 logger.info("Job %r is done", job)
                 job_queue.complete(job)
             else:

demo.py

@@ -1,5 +1,15 @@
+import logging
+import os
+from pathlib import Path
+
 from openwpm.command_sequence import CommandSequence
 from openwpm.config import BrowserParams, ManagerParams
+from openwpm.storage.cloud_storage.gcp_storage import (
+    GcsStructuredProvider,
+    GcsUnstructuredProvider,
+)
+from openwpm.storage.local_storage import LocalArrowProvider
+from openwpm.storage.sql_provider import SqlLiteStorageProvider
 from openwpm.task_manager import TaskManager
 
 # The list of sites that we wish to crawl
@@ -35,26 +45,35 @@
     browser_params[i].dns_instrument = True
 
 # Update TaskManager configuration (use this for crawl-wide settings)
-manager_params.data_directory = "~/Desktop/"
-manager_params.log_directory = "~/Desktop/"
+manager_params.data_directory = Path("./datadir/")
+manager_params.log_directory = Path("./datadir/")
 
+logging_params = {"log_level_console": logging.DEBUG}
 # memory_watchdog and process_watchdog are useful for large scale cloud crawls.
 # Please refer to docs/Configuration.md#platform-configuration-options for more information
 # manager_params.memory_watchdog = True
 # manager_params.process_watchdog = True
 
 # Instantiates the measurement platform
+project = "senglehardt-openwpm-test-1"
+bucket_name = "openwpm-test-bucket"
 # Commands time out by default after 60 seconds
-manager = TaskManager(manager_params, browser_params)
+manager = TaskManager(
+    manager_params,
+    browser_params,
+    GcsStructuredProvider(project, bucket_name, base_path="test3"),
+    None,
+)
 
 # Visits the sites
-for site in sites:
+for index, site in enumerate(sites):
+
+    def callback(success: bool, val: str = site) -> None:
+        print("CommandSequence {} done".format(val))
 
     # Parallelize sites over all number of browsers set above.
     command_sequence = CommandSequence(
-        site,
-        reset=True,
-        callback=lambda success, val=site: print("CommandSequence {} done".format(val)),
+        site, site_rank=index, reset=True, callback=callback,
     )
 
     # Start by visiting the page

environment.yaml

@@ -7,37 +7,33 @@ dependencies:
 - click=7.1.2
 - codecov=2.1.10
 - dill=0.3.3
+- gcsfs=0.7.1
 - geckodriver=0.28.0
 - ipython=7.19.0
 - leveldb=1.22
-- localstack=0.11.1.1
 - multiprocess=0.70.11.1
+- mypy=0.790
 - nodejs=14.15.1
-- pandas=1.1.4
+- pandas=1.1.5
 - pillow=8.0.1
-- pip=20.2.4
-- pre-commit=2.9.2
+- pip=20.3.1
+- pre-commit=2.9.3
 - psutil=5.7.3
 - pyarrow=2.0.0
 - pytest-cov=2.10.1
 - pytest=6.1.2
 - python=3.8.6
 - pyvirtualdisplay=0.2.5
 - redis-py=3.5.3
-- s3fs=0.4.0
+- s3fs=0.5.1
 - selenium=3.141.0
-- sentry-sdk=0.19.4
+- sentry-sdk=0.19.5
 - tabulate=0.8.7
 - tblib=1.6.0
 - wget=1.20.1
 - pip:
-  - amazon-kclpy==2.0.1
-  - crontab==0.22.9
   - dataclasses-json==0.5.2
   - domain-utils==0.7.1
-  - flask-cors==3.0.9
   - jsonschema==3.2.0
-  - moto-ext==1.3.15.15
   - plyvel==1.3.0
-  - subprocess32==3.5.4
 name: openwpm