Permissions rework

.circleci/config.yml

@@ -177,7 +177,7 @@ jobs:
       - attach_workspace:
           at: /home/circleci/
       - restore_cache:
-          key: flowdb-deps-4-{{ checksum "flowdb/tests/Pipfile.lock"}}
+          key: flowdb-deps-6-{{ checksum "flowdb/tests/Pipfile.lock"}}
       - run:
           name: "Switch to Python v3.9.4"
           command: |
@@ -191,7 +191,7 @@ jobs:
           command: |
             PIPENV_PIPFILE=flowdb/tests/Pipfile pipenv install --deploy --dev
       - save_cache:
-          key: flowdb-deps-4-{{ checksum "flowdb/tests/Pipfile.lock" }}
+          key: flowdb-deps-6-{{ checksum "flowdb/tests/Pipfile.lock" }}
           paths:
             - /home/circleci/.local/share/virtualenvs/
       - run:
@@ -386,13 +386,13 @@ jobs:
     steps:
       - checkout
       - restore_cache:
-          key: flowmachine-deps-5-{{ checksum "flowmachine/Pipfile.lock" }}
+          key: flowmachine-deps-6-{{ checksum "flowmachine/Pipfile.lock" }}
       # Need to install graphviz and pygraphviz manually because it was removed from the Pipfile
       # (see https://github.com/Flowminder/FlowKit/issues/952)
       - run: cd flowmachine && pipenv install --dev --deploy && pipenv run pip install -e .
       - run: cd flowmachine && sudo apt-get update && sudo apt-get install -y libgraphviz-dev graphviz && pipenv run pip install pygraphviz
       - save_cache:
-          key: flowmachine-deps-5-{{ checksum "flowmachine/Pipfile.lock" }}
+          key: flowmachine-deps-6-{{ checksum "flowmachine/Pipfile.lock" }}
           paths:
             - /home/circleci/.local/share/virtualenvs/flowmachine-caaCcVrN
 
@@ -402,7 +402,7 @@ jobs:
     steps:
       - checkout
       - restore_cache:
-          key: flowmachine-deps-5-{{ checksum "flowmachine/Pipfile.lock" }}
+          key: flowmachine-deps-6-{{ checksum "flowmachine/Pipfile.lock" }}
       - run:
           name: Linting files with black
           # Installed the version of Black from flowmachine's lockfile
@@ -417,12 +417,12 @@ jobs:
           path: /home/circleci/project/
       - restore_cache:
           keys:
-            - flowauth-deps-4-{{ checksum "package.json" }}
+            - flowauth-deps-6-{{ checksum "package.json" }}
       - run:
           name: Install deps
           command: npm ci
       - save_cache:
-          key: flowauth-deps-4-{{ checksum "package.json" }}
+          key: flowauth-deps-6-{{ checksum "package.json" }}
           # cache NPM modules and the folder with the Cypress binary
           paths:
             - ~/.npm
@@ -443,7 +443,7 @@ jobs:
       - attach_workspace:
           at: /home/circleci/
       - restore_cache:
-          key: flowmachine-deps-5-{{ checksum "Pipfile.lock" }}
+          key: flowmachine-deps-6-{{ checksum "Pipfile.lock" }}
       - run:
           name: Install graphviz
           command: sudo apt-get update && sudo apt-get install -y xvfb libgraphviz-dev graphviz
@@ -568,7 +568,7 @@ jobs:
 
   run_flowauth_end_to_end_tests:
     docker:
-      - image: cypress/base
+      - image: cypress/base:16.13.0
       - image: flowminder/flowauth:$CIRCLE_SHA1
         environment:
           DB_URI: postgresql://flowauth:{}@localhost:5432/flowauth
@@ -591,12 +591,12 @@ jobs:
           path: /home/circleci/project/
       - restore_cache:
           keys:
-            - flowauth-deps-4-{{ checksum "package.json" }}
+            - flowauth-deps-6-{{ checksum "package.json" }}
       - run:
           name: Cypress setup
           command: npm ci
       - save_cache:
-          key: flowauth-deps-4-{{ checksum "package.json" }}
+          key: flowauth-deps-6-{{ checksum "package.json" }}
           # cache NPM modules and the folder with the Cypress binary
           paths:
             - ~/.npm
@@ -609,6 +609,9 @@ jobs:
       - run:
           name: Run Cypress end-to-end tests
           command: DEBUG="cypress:*" $(npm bin)/cypress run --record --reporter junit --reporter-options "mochaFile=results/flowauth-frontend.[hash].xml"
+      - run:
+          name: Run Cypress component tests
+          command: DEBUG="cypress:*" $(npm bin)/cypress run --component --record --reporter junit --reporter-options "mochaFile=results/flowauth-component.[hash].xml"
       - store_test_results:
           path: results
       - store_artifacts:
@@ -643,7 +646,7 @@ jobs:
           path: /home/circleci/project/
       - restore_cache:
           keys:
-            - autoflow-deps-4-{{ checksum "Pipfile.lock" }}
+            - autoflow-deps-6-{{ checksum "Pipfile.lock" }}
       - run: &install_autoflow_deps
           name: Install non-python autoflow dependencies
           command: |
@@ -656,7 +659,7 @@ jobs:
           command: |
             pipenv install --deploy --dev
       - save_cache:
-          key: autoflow-deps-4-{{ checksum "Pipfile.lock" }}
+          key: autoflow-deps-6-{{ checksum "Pipfile.lock" }}
           paths:
             - /home/circleci/.local/share/virtualenvs/
       - run:
@@ -715,7 +718,7 @@ jobs:
       - checkout:
           path: /home/circleci/project/
       - restore_cache:
-          key: flowetl-unit-deps-5-{{ checksum "Pipfile.lock"}}
+          key: flowetl-unit-deps-6-{{ checksum "Pipfile.lock"}}
       - run:
           name: Install pipenv
           command: pip install --upgrade pipenv
@@ -724,7 +727,7 @@ jobs:
           command: |
             pipenv install --deploy --dev
       - save_cache:
-          key: flowetl-unit-deps-5-{{ checksum "Pipfile.lock" }}
+          key: flowetl-unit-deps-6-{{ checksum "Pipfile.lock" }}
           paths:
             - /home/circleci/.local/share/virtualenvs/
       - run:
@@ -749,7 +752,7 @@ jobs:
       - checkout:
           path: /home/circleci/project/
       - restore_cache:
-          key: flowetl-deps-5-{{ checksum "Pipfile.lock"}}
+          key: flowetl-deps-6-{{ checksum "Pipfile.lock"}}
       - run:
           name: Install psycopg2 build deps
           command: |
@@ -772,7 +775,7 @@ jobs:
           command: |
             sudo apt-get update && sudo apt-get install -y postgresql
       - save_cache:
-          key: flowetl-deps-5-{{ checksum "Pipfile.lock" }}
+          key: flowetl-deps-6-{{ checksum "Pipfile.lock" }}
           paths:
             - /home/circleci/.local/share/virtualenvs/
       - run:
@@ -803,15 +806,15 @@ jobs:
       - attach_workspace:
           at: /home/circleci
       - restore_cache:
-          key: integration-test-deps-4-{{ checksum "Pipfile.lock" }}
+          key: integration-test-deps-6-{{ checksum "Pipfile.lock" }}
       - when:
           condition:
             equal: ["not query_tests", << parameters.pytest_selector >>]
           steps:
             - run: *install_autoflow_deps
       - run: pipenv install --deploy --dev
       - save_cache:
-          key: integration-test-deps-4-{{ checksum "Pipfile.lock" }}
+          key: integration-test-deps-6-{{ checksum "Pipfile.lock" }}
           paths:
             - /home/circleci/.local/share/virtualenvs/
       - run: *wait_for_flowdb
@@ -882,7 +885,7 @@ jobs:
       - attach_workspace:
           at: /home/circleci/
       - restore_cache:
-          key: flowkit-docs-deps-5-{{ checksum "Pipfile.lock" }}
+          key: flowkit-docs-deps-6-{{ checksum "Pipfile.lock" }}
       - run:
           name: Install pandoc
           command: |
@@ -976,7 +979,7 @@ jobs:
           path: /home/circleci/project/docs/pg_log.zip
           destination: pg_log
       - save_cache:
-          key: flowkit-docs-deps-5-{{ checksum "Pipfile.lock" }}
+          key: flowkit-docs-deps-6-{{ checksum "Pipfile.lock" }}
           paths:
             - /home/circleci/.local/share/virtualenvs/
 

.gitignore

@@ -132,3 +132,4 @@ docs/source/_static/openapi-redoc.json
 *.received.txt
 
 secrets_quickstart/flowdb_pgdata
+local_ci_process.yml

CHANGELOG.md

@@ -10,8 +10,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Added views `etl.ingested_state`, `etl.available_dates` and `etl.deduped_post_etl_queries` in FlowDB, for convenient extraction of relevant information from the ETL tables. [#5641](https://github.com/Flowminder/FlowKit/issues/5641)
 
 ### Changed
+- *Important*; tokens issued by previous versions of Flowauth are not compatible with this version. Users will need to regenerate tokens using the updated Flowauth.
+- Move from `groups` to `roles` in flowauth; see [here](https://github.com/Flowminder/FlowKit/pull/5163#issuecomment-1216480419) for full details. [#5613](https://github.com/Flowminder/FlowKit/pull/5163)
 - Changed `AIRFLOW__CORE__SQL_ALCHEMY_CONN` env var to `AIRFLOW__DATABASE__SQL_ALCHEMY_CONN`
+- RoleScopePicker component redesigned and reimplemented.
 - Docs now recommend creating a separate bind mount for airflow scheduler logs, and include this in the secrets quickstart. [#3622](https://github.com/Flowminder/FlowKit/issues/3622)
+- `jwt` tokens now use `sub` instead of `identity` for `JWT_IDENTITY_CLAIM`.
+
 
 ### Fixed
 - Fixed a potential deadlock when using a small connection pool and `store`-ing queries

docs/notebook_preamble.py

@@ -47,6 +47,8 @@ def format_dict(x):
     username="docsuser",
     private_key=load_private_key(os.environ["PRIVATE_JWT_SIGNING_KEY"]),
     lifetime=timedelta(days=1),
-    claims=get_all_claims_from_flowapi(flowapi_url="http://localhost:9090"),
+    roles=dict(
+        universal_role=get_all_claims_from_flowapi(flowapi_url="http://localhost:9090")
+    ),
     flowapi_identifier=os.environ["FLOWAPI_IDENTIFIER"],
 )

docs/source/developer/adr/0012-claims-role-scope-rework.md

@@ -0,0 +1,74 @@
+# Rework of permissions system
+
+2022/12/16
+
+**Status**
+
+Accepted
+
+**Context**
+
+When a user passes their query to a Flowmachine instance, they first sign into a Flowauth instance. Within a Flowauth instance, a user has a set of claims that define what combination of queries can be run.
+The space for these scopes was generated by a full tree walk of every combination of a query, its geographic components, and all sub-queries (and their geographic components) - though normally derived through targetted traversal of the query tree, there were places where a full walk was required. This offered a very fine degree of control over what queries you could permit users to run, but as the available set of queries increased this space expanded exponentially, eventually becoming unwieldy.
+This manifested in several places:
+ - When building the API spec inside flowAPI
+ - When displaying the spec on the Flowauth frontend for servers (leading to timeouts and crashes on some lower-capability machines)
+ - When passing large or specific scopes around as JWTs
+
+**Decision**
+
+
+The core of the change that has been implemented is to change the definition of what a user can access. Instead of a combination of queries, users are now assigned `roles`, which are composed of `scopes`. A scope can be one of two types;
+ - *Simple scopes*
+
+   These control access to a server-wide capability and are all-or-nothing. The currently implemented simple scopes are:
+   - `run`; permits a user that holds this role to run queries using the `run` flowAPI endpoint
+   - `get_result`; permits a user to get the results of a query using the `get` flowAPI endpoint
+   - `get_available_dates`; permits a user to get the list of dates available to run queries against on a server.
+
+   These scopes do *not* imply that a role provided the capabilites to run a query by themselves - they must be provided in combination with one or more complex scopes.
+
+ - *Complex scopes*
+
+   These control access to a combination of a geographic component, a top-level query and a descendent query.
+   - Top-level queries are the methods that are available to the users of Flowmachine directly.
+   - Each top-level query may require a set of sub-queries to run, wich in turn may require sub-queries of their own; these are the descendent queries.
+   - Finally, the geographic component is the spatial presentation that is available to the role. Queries without spatial aggregates (such as `historgram_aggregate`) have the geographic component `nonspatial`.
+
+   Complex scopes are of the form `geographic_component:top_level_query:sub_query` - this ordering has been built on assumed order of importance to users. For example it will be a more common use case that an administrator will want to create a role that restrics access only to admin levels 0 or 1 than to the `most_frequest_location` sub-query of the `spatial_aggregate` top-level query.
+
+As a consequence of this, the new Flowauth db schema is shown below:
+
+```mermaid
+erDiagram
+    Servers ||--|{ Roles : contains
+    Roles }|--|{ Scopes : "provides permissions for"
+    Users }|--|{ Roles : "can work with"
+    Servers ||--|{ Scopes : "provides"
+```
+
+And the new auth flow is as follows:
+
+
+```mermaid
+sequenceDiagram
+    participant Flowauth
+    actor User
+    participant FlowAPI
+    note over User: Selects roles for token
+    User ->>+ Flowauth: roles
+    note over Flowauth: Checks user is permitted roles
+    Flowauth ->>+ User: signed token w/ roles + scopes
+    note over User: Builds query
+    User ->>+ FlowAPI: query, token
+    note over FlowAPI: Checks query params are covered by a single role
+    FlowAPI ->>+ User: query results
+```
+
+**Consequences**
+ - Much faster and more lightweight building of tokens
+ - Flowauth frontend can now be used on lower-capability machines
+ - Less fine-grained control over query permissions
+    - This does have the knock-on effect of reducing the decision space to a more managable size,perhaps leading users to make meaningful decisions about scopes within a role instead of defaulting to 'all'.
+ - Users can't be explicitly grouped anymore (roles supercede group functions)
+ - Scopes cannot be assigned directly to users anymore.