Data Aggregator Rewrite

.github/workflows/run-tests.yaml

@@ -0,0 +1,98 @@
+# This workflow will run all tests as well as pre-commit 
+
+name: Tests and linting
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  schedule:
+    - cron: '0 0 */2 * *'
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    # All of these steps are just setup
+    - uses: actions/checkout@v2
+    - name: Setting MINICONDA_PATH
+      run: echo "MINICONDA_PATH=$HOME/miniconda" >> $GITHUB_ENV
+    - name: Setting OPENWPM_CONDA_PATH
+      run: echo "OPENWPM_CONDA_PATH=$MINICONDA_PATH/envs/openwpm" >> $GITHUB_ENV
+    # If the environment.yaml hasn't changed we just reuse the entire conda install
+    - id: cache
+      uses: actions/cache@v2
+      env:
+        cache-name: conda-cache
+      with:
+        path: ${{ env.MINICONDA_PATH }}
+        key: ${{ env.cache-name }}-${{ hashFiles('environment.yaml') }}
+
+    - name: Install conda
+      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      run: $GITHUB_WORKSPACE/scripts/install-miniconda.sh
+
+    - run: echo "$MINICONDA_PATH/bin" >> $GITHUB_PATH
+
+    - name: Install.sh (cache miss)
+      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      run: $GITHUB_WORKSPACE/install.sh
+    - name: Install.sh (cache hit)
+      if: ${{ steps.cache.outputs.cache-hit == 'true' }}
+      run: $GITHUB_WORKSPACE/install.sh --skip-create
+    - run: echo "$OPENWPM_CONDA_PATH/bin" >> $GITHUB_PATH
+    # Now we have a working OpenWPM environment
+
+    - run: pre-commit run --all
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-groups: ["test/test_[a-e]*", "test/test_[f-h]*", "test/test_[i-r,t-z]*", "test/test_[s]*", "test/storage/*"]
+      fail-fast: false
+    steps:
+    # All of these steps are just setup, maybe we should wrap them in an action
+    - uses: actions/checkout@v2
+    - name: Cache node modules
+      uses: actions/cache@v2
+      env:
+        cache-name: cache-node-modules
+      with:
+        # npm cache files are stored in `~/.npm` on Linux/macOS
+        path: ~/.npm
+        key: ${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+    # Setting env variables that depend on $HOME
+    - name: Setting MINICONDA_PATH
+      run: echo "MINICONDA_PATH=$HOME/miniconda" >> $GITHUB_ENV
+    - name: Setting OPENWPM_CONDA_PATH
+      run: echo "OPENWPM_CONDA_PATH=$MINICONDA_PATH/envs/openwpm" >> $GITHUB_ENV
+
+    # If the environment.yaml hasn't changed we just reuse the entire conda install
+    - id: conda-cache
+      uses: actions/cache@v2
+      env:
+        cache-name: conda-cache
+      with:
+        path: ${{ env.MINICONDA_PATH }}
+        key: ${{ env.cache-name }}-${{ hashFiles('environment.yaml') }}
+
+    - name: Install conda
+      if: ${{ steps.conda-cache.outputs.cache-hit != 'true' }}
+      run: $GITHUB_WORKSPACE/scripts/install-miniconda.sh
+
+    - run: echo "$MINICONDA_PATH/bin" >> $GITHUB_PATH
+
+    - name: Install.sh (cache miss)
+      if: ${{ steps.conda-cache.outputs.cache-hit != 'true' }}
+      run: $GITHUB_WORKSPACE/install.sh
+    - name: Install.sh (cache hit)
+      if: ${{ steps.conda-cache.outputs.cache-hit == 'true' }}
+      run: $GITHUB_WORKSPACE/install.sh --skip-create
+
+    - run: echo "$OPENWPM_CONDA_PATH/bin" >> $GITHUB_PATH
+    # Now we have a working OpenWPM environment
+
+    - run: ./scripts/ci.sh
+      env:
+        DISPLAY: ":99.0"
+        TESTS: ${{ matrix.test-groups }}

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
     -   id: isort
   - repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
       - id: black
         language_version: python3

CONTRIBUTING.md

@@ -85,7 +85,6 @@ OpenWPM's tests are build on [pytest](https://docs.pytest.org/en/latest/). Execu
 in the test directory to run all tests:
 
     $ conda activate openwpm
-    $ cd test
     $ py.test -vv
 
 See the [pytest docs](https://docs.pytest.org/en/latest/) for more information on selecting

README.md

@@ -1,6 +1,6 @@
 
 OpenWPM
-[![Build Status](https://travis-ci.org/mozilla/OpenWPM.svg?branch=master)](https://travis-ci.org/mozilla/OpenWPM)
+[![Build Status](https://github.com/mozilla/openwpm/workflows/Tests%20and%20linting/badge.svg?branch=master)](https://github.com/mozilla/openwpm/actions?query=branch%3Amaster)
 [![OpenWPM Matrix Channel](https://img.shields.io/matrix/OpenWPM:mozilla.org?label=Join%20us%20on%20matrix&server_fqdn=mozilla.modular.im)](https://matrix.to/#/#OpenWPM:mozilla.org?via=mozilla.org) <!-- omit in toc -->
 =======
 
@@ -12,25 +12,25 @@ the instrumentation section below for more details.
 
 Table of Contents <!-- omit in toc -->
 ------------------
-* [Installation](#installation)
-  * [Pre-requisites](#pre-requisites)
-  * [Install](#install)
-  * [Mac OSX](#mac-osx)
-* [Quick Start](#quick-start)
-* [Troubleshooting](#troubleshooting)
-* [Advice for Measurement Researchers](#advice-for-measurement-researchers)
-* [Developer instructions](#developer-instructions)
-* [Instrumentation and Configuration](#instrumentation-and-configuration)
-* [Persistence Types](#persistence-types)
-    * [Local Databases](#local-databases)
-    * [Parquet on Amazon S3](#parquet-on-amazon-s3)
-* [Docker Deployment for OpenWPM](#docker-deployment-for-openwpm)
-  * [Building the Docker Container](#building-the-docker-container)
-  * [Running Measurements from inside the Container](#running-measurements-from-inside-the-container)
-  * [MacOS GUI applications in Docker](#macos-gui-applications-in-docker)
-* [Citation](#citation)
-* [License](#license)
 
+- [Installation](#installation)
+  - [Pre-requisites](#pre-requisites)
+  - [Install](#install)
+  - [Mac OSX](#mac-osx)
+- [Quick Start](#quick-start)
+- [Troubleshooting](#troubleshooting)
+- [Advice for Measurement Researchers](#advice-for-measurement-researchers)
+- [Developer instructions](#developer-instructions)
+- [Instrumentation and Configuration](#instrumentation-and-configuration)
+- [Storage](#storage)
+  - [Local Storage](#local-storage)
+  - [Remote storage](#remote-storage)
+- [Docker Deployment for OpenWPM](#docker-deployment-for-openwpm)
+  - [Building the Docker Container](#building-the-docker-container)
+  - [Running Measurements from inside the Container](#running-measurements-from-inside-the-container)
+  - [MacOS GUI applications in Docker](#macos-gui-applications-in-docker)
+- [Citation](#citation)
+- [License](#license)
 
 Installation
 ------------
@@ -82,8 +82,8 @@ Quick Start
 
 Once installed, it is very easy to run a quick test of OpenWPM. Check out
 `demo.py` for an example. This will use the default setting specified in
-`openwpm/default_manager_params.json` and
-`openwpm/default_browser_params.json`, with the exception of the changes
+`openwpm/config.py::ManagerParams` and
+`openwpm/config.py::BrowserParams`, with the exception of the changes
 specified in `demo.py`.
 
 More information on the instrumentation and configuration parameters is given
@@ -178,40 +178,52 @@ If you want to contribute to OpenWPM have a look at our [CONTRIBUTING.md](./CONT
 
 Instrumentation and Configuration
 -------------------------------
+
 OpenWPM provides a breadth of configuration options which can be found
 in [Configuration.md](docs/Configuration.md)
 More detail on the output is available [below](#persistence-types).
 
-Persistence Types
+Storage
 ------------
 
-#### Local Databases
-By default OpenWPM saves all data locally on disk in a variety of formats.
-Most of the instrumentation saves to a SQLite database specified
-by `manager_params.database_name` in the main output directory. Response
-bodies are saved in a LevelDB database named `content.ldb`, and are keyed by
-the hash of the content. In addition, the browser commands that dump page
-source and save screenshots save them in the `sources` and `screenshots`
-subdirectories of the main output directory. The SQLite schema
-specified by: `openwpm/DataAggregator/schema.sql`. You can specify additional tables
-inline by sending a `create_table` message to the data aggregator.
-
-#### Parquet on Amazon S3
-As an option, OpenWPM can save data directly to an Amazon S3 bucket as a
-Parquet Dataset. This is currently experimental and hasn't been thoroughly
-tested. Screenshots, and page source saving is not currently supported and
-will still be stored in local databases and directories. To enable S3
-saving specify the following configuration parameters in `manager_params`:
-* Persistence Type: `manager_params.output_format = 's3'`
-* S3 bucket name: `manager_params.s3_bucket = 'openwpm-test-crawl'`
-* Directory within S3 bucket: `manager_params.s3_directory = '2018-09-09_test-crawl-new'`
-
-In order to save to S3 you must have valid access credentials stored in
-`~/.aws`. We do not currently allow you to specify an alternate storage
-location.
-
-**NOTE:** The schemas should be kept in sync with the exception of
-output-specific columns (e.g., `instance_id` in the S3 output). You can compare
+OpenWPM distinguishes between two types of data, structured and unstructured.
+Structured data is all data captured by the instrumentation or emitted by the platform.
+Generally speaking all data you download is unstructured data.
+
+For each of the data classes we offer a variety of storage providers, and you are encouraged
+to implement your own, should the provided backends not be enough for you.
+
+We have an outstanding issue to enable saving content generated by commands, such as
+screenshots and page dumps to unstructured storage (see [#232](https://github.com/mozilla/OpenWPM/issues/232)).  
+For now, they get saved to `manager_params.data_directory`.
+
+### Local Storage
+
+For storing structured data locally we offer two StorageProviders:
+
+- The SQLiteStorageProvider which writes all data into a SQLite database
+  - This is the recommended approach for getting started as the data is easily explorable
+- The LocalArrowProvider which stores the data into Parquet files.
+  - This method integrates well with NumPy/Pandas
+  - It might be harder to ad-hoc process
+
+For storing unstructured data locally we also offer two solutions:
+
+- The LevelDBProvider which stores all data into a LevelDB
+  - This is the recommended approach
+- The LocalGzipProvider that gzips and stores the files individually on disk
+  - Please note that file systems usually don't like thousands of files in one folder
+  - Use with care or for single site visits
+
+### Remote storage
+
+When running in the cloud, saving records to disk is not a reasonable thing to do.
+So we offer a remote StorageProviders for S3 (See [#823](https://github.com/mozilla/OpenWPM/issues/823)) and GCP.
+Currently, all remote StorageProviders write to the respective object storage service (S3/GCS).
+The structured providers use the Parquet format.
+
+**NOTE:** The Parquet and SQL schemas should be kept in sync except
+output-specific columns (e.g., `instance_id` in the Parquet output). You can compare
 the two schemas by running
 `diff -y openwpm/DataAggregator/schema.sql openwpm/DataAggregator/parquet_schema.py`.
 
@@ -238,7 +250,7 @@ Docker service.
 __Step 2:__ to build the image, run the following command from a terminal
 within the root OpenWPM directory:
 
-```
+```bash
     docker build -f Dockerfile -t openwpm .
 ```
 
@@ -253,7 +265,7 @@ X-server. You can do this by running: `xhost +local:docker`
 
 Then you can run the demo script using:
 
-```
+```bash
     mkdir -p docker-volume && docker run -v $PWD/docker-volume:/opt/Desktop \
     -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix --shm-size=2g \
     -it openwpm python3 /opt/OpenWPM/demo.py

crawler.py

@@ -31,7 +31,7 @@
 # Storage Provider Params
 CRAWL_DIRECTORY = os.getenv("CRAWL_DIRECTORY", "crawl-data")
 GCS_BUCKET = os.getenv("GCS_BUCKET", "openwpm-crawls")
-GCP_PROJECT = os.getenv("GCP_PROJECT", "senglehardt-openwpm-test-1")
+GCP_PROJECT = os.getenv("GCP_PROJECT", "")
 AUTH_TOKEN = os.getenv("GCP_AUTH_TOKEN", "cloud")
 
 # Browser Params
@@ -87,9 +87,6 @@
 # Manager configuration
 manager_params.data_directory = Path("~/Desktop/") / CRAWL_DIRECTORY
 manager_params.log_directory = Path("~/Desktop/") / CRAWL_DIRECTORY
-manager_params.output_format = "s3"
-manager_params.s3_bucket = GCS_BUCKET
-manager_params.s3_directory = CRAWL_DIRECTORY
 
 structured = GcsStructuredProvider(
     project=GCP_PROJECT,
@@ -100,7 +97,7 @@
 unstructured = GcsUnstructuredProvider(
     project=GCP_PROJECT,
     bucket_name=GCS_BUCKET,
-    base_path=CRAWL_DIRECTORY,
+    base_path=CRAWL_DIRECTORY + "/data",
     token=AUTH_TOKEN,
 )
 # Instantiates the measurement platform
@@ -119,7 +116,7 @@
     with sentry_sdk.configure_scope() as scope:
         # tags generate breakdown charts and search filters
         scope.set_tag("CRAWL_DIRECTORY", CRAWL_DIRECTORY)
-        scope.set_tag("S3_BUCKET", GCS_BUCKET)
+        scope.set_tag("GCS_BUCKET", GCS_BUCKET)
         scope.set_tag("DISPLAY_MODE", DISPLAY_MODE)
         scope.set_tag("HTTP_INSTRUMENT", HTTP_INSTRUMENT)
         scope.set_tag("COOKIE_INSTRUMENT", COOKIE_INSTRUMENT)
@@ -136,7 +133,10 @@
         if PREFS:
             scope.set_context("PREFS", json.loads(PREFS))
         scope.set_context(
-            "crawl_config", {"REDIS_QUEUE_NAME": REDIS_QUEUE_NAME,},
+            "crawl_config",
+            {
+                "REDIS_QUEUE_NAME": REDIS_QUEUE_NAME,
+            },
         )
     # Send a sentry error message (temporarily - to easily be able
     # to compare error frequencies to crawl worker instance count)