Add a flag to the pull command that pulls all resources from the server

.fides/dataset.yml

@@ -1,6 +1,7 @@
 dataset:
 - fides_key: public
   organization_fides_key: default_organization
+  tags: null
   name: public
   description: The dataset responsible for storing all of the data related to a fidesctl
     instance.
@@ -198,14 +199,14 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
+    - name: tags
       description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
+    - name: updated_at
       description: null
       data_categories:
       - system.operations
@@ -267,14 +268,14 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
+    - name: tags
       description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
+    - name: updated_at
       description: null
       data_categories:
       - system.operations
@@ -345,14 +346,14 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
+    - name: tags
       description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
+    - name: updated_at
       description: null
       data_categories:
       - system.operations
@@ -450,14 +451,14 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
+    - name: tags
       description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
+    - name: updated_at
       description: null
       data_categories:
       - system.operations
@@ -585,22 +586,22 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: third_country_transfers
+    - name: tags
       description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
-      description: The timestamp of when the row was last updated
+    - name: third_country_transfers
+      description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
-      description: null
+    - name: updated_at
+      description: The timestamp of when the row was last updated
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
@@ -878,15 +879,15 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
-      description: The timestamp of when the row was last updated
+    - name: tags
+      description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
-      description: null
+    - name: updated_at
+      description: The timestamp of when the row was last updated
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
@@ -947,15 +948,15 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
-      description: The timestamp of when the row was last updated
+    - name: tags
+      description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
-      description: null
+    - name: updated_at
+      description: The timestamp of when the row was last updated
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
@@ -1009,15 +1010,15 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
-      description: The timestamp of when the row was last updated
+    - name: tags
+      description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
-      description: null
+    - name: updated_at
+      description: The timestamp of when the row was last updated
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
@@ -1143,22 +1144,22 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: third_country_transfers
+    - name: tags
       description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: updated_at
-      description: The timestamp of when the row was last updated
+    - name: third_country_transfers
+      description: null
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
-    - name: tags
-      description: null
+    - name: updated_at
+      description: The timestamp of when the row was last updated
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

.fides/policy.yml

@@ -1,6 +1,7 @@
 policy:
 - fides_key: fidesctl_policy
   organization_fides_key: default_organization
+  tags: null
   name: Fidesctl Policy
   description: The main privacy policy for Fidesctl.
   rules:
@@ -20,6 +21,7 @@ policy:
     data_qualifier: aggregated
 - fides_key: data_sharing_policy
   organization_fides_key: default_organization
+  tags: null
   name: Data Sharing
   description: The privacy policy that governs sharing of data with third parties.
   rules:

CHANGELOG.md

@@ -28,7 +28,8 @@ The types of changes are:
   * System scanning results: AWS systems are stored and can be selected for review
 * Added Cypress for testing [713](https://github.com/ethyca/fides/pull/833)
 * CustomInput type "password" with show/hide icon.
-* Sync CLI command now checks for untracked/unstaged files in the manifests dir [#869](https://github.com/ethyca/fides/pull/869)
+* Pull CLI command now checks for untracked/unstaged files in the manifests dir [#869](https://github.com/ethyca/fides/pull/869)
+* Pull CLI command has a flag to pull missing files from the server [#895](https://github.com/ethyca/fides/pull/895)
 * Add Okta support to the `/generate` endpoint [#842](https://github.com/ethyca/fides/pull/842)
 * Add db support to `/generate` endpoint [849](https://github.com/ethyca/fides/pull/849)
 * Add BigQuery support for the `generate` cli command [#814](https://github.com/ethyca/fides/pull/814)

src/fidesctl/cli/__init__.py

@@ -11,7 +11,7 @@
 from fidesctl.core.config import get_config
 
 from .commands.annotate import annotate
-from .commands.core import apply, evaluate, parse, sync
+from .commands.core import apply, evaluate, parse, pull
 from .commands.crud import delete, get, ls
 from .commands.db import database
 from .commands.export import export
@@ -34,7 +34,7 @@
     get,
     ls,
     status,
-    sync,
+    pull,
 ]
 API_COMMAND_DICT = {command.name or str(command): command for command in API_COMMANDS}
 ALL_COMMANDS = API_COMMANDS + LOCAL_COMMANDS

src/fidesctl/cli/commands/core.py

@@ -1,4 +1,6 @@
 """Contains all of the core CLI commands for Fidesctl."""
+from typing import Optional
+
 import click
 
 from fidesctl.cli.options import (
@@ -12,7 +14,7 @@
 from fidesctl.core import audit as _audit
 from fidesctl.core import evaluate as _evaluate
 from fidesctl.core import parse as _parse
-from fidesctl.core import sync as _sync
+from fidesctl.core import pull as _pull
 from fidesctl.core.utils import git_is_dirty
 
 
@@ -138,24 +140,34 @@ def parse(ctx: click.Context, manifests_dir: str, verbose: bool = False) -> None
 @click.command()
 @click.pass_context
 @manifests_dir_argument
+@click.option(
+    "--all-resources",
+    "-a",
+    default=None,
+    help="Pulls all locally missing resources from the server into this file.",
+)
 @with_analytics
-def sync(ctx: click.Context, manifests_dir: str) -> None:
+def pull(ctx: click.Context, manifests_dir: str, all_resources: Optional[str]) -> None:
     """
     Update local resource files by their fides_key to match their server versions.
 
-    Aborts the sync if there are unstaged or untracked files in the manifests dir.
+    Alternatively, with the "--all" flag all resources from the server will be pulled
+    down into a local file.
+
+    The pull is aborted if there are unstaged or untracked files in the manifests dir.
     """
 
     config = ctx.obj["CONFIG"]
-    # Do this to validate the manifests since they won't get parsed during the sync process
+    # Do this to validate the manifests since they won't get parsed during the pull process
     _parse.parse(manifests_dir)
     if git_is_dirty(manifests_dir):
         echo_red(
-            f"There are unstaged changes in your manifest directory: '{manifests_dir}' \nAborting sync!"
+            f"There are unstaged changes in your manifest directory: '{manifests_dir}' \nAborting pull!"
         )
         raise SystemExit(1)
-    _sync.sync(
+    _pull.pull(
         url=config.cli.server_url,
         manifests_dir=manifests_dir,
         headers=config.user.request_headers,
+        all_resources=all_resources,
     )

src/fidesctl/cli/commands/util.py

@@ -7,7 +7,12 @@
 from fideslog.sdk.python.utils import OPT_OUT_COPY, OPT_OUT_PROMPT
 
 import fidesctl
-from fidesctl.cli.utils import check_server, send_init_analytics, with_analytics
+from fidesctl.cli.utils import (
+    FIDESCTL_ASCII_ART,
+    check_server,
+    send_init_analytics,
+    with_analytics,
+)
 from fidesctl.core.utils import echo_green
 
 
@@ -45,7 +50,7 @@ def init(ctx: click.Context, fides_directory_location: str) -> None:
         "cli": {"server_protocol", "server_host", "server_port", "analytics_id"},
         "user": {"analytics_opt_out"},
     }
-
+    click.echo(FIDESCTL_ASCII_ART)
     click.echo("Initializing Fidesctl...")
     separate()
 

src/fidesctl/cli/utils.py

@@ -34,6 +34,15 @@
 from fidesctl.core.config.utils import get_config_from_file, update_config_file
 from fidesctl.core.utils import API_PREFIX, check_response, echo_green, echo_red
 
+FIDESCTL_ASCII_ART = """
+███████╗██╗██████╗ ███████╗███████╗ ██████╗████████╗██╗     
+██╔════╝██║██╔══██╗██╔════╝██╔════╝██╔════╝╚══██╔══╝██║     
+█████╗  ██║██║  ██║█████╗  ███████╗██║        ██║   ██║     
+██╔══╝  ██║██║  ██║██╔══╝  ╚════██║██║        ██║   ██║     
+██║     ██║██████╔╝███████╗███████║╚██████╗   ██║   ███████╗
+╚═╝     ╚═╝╚═════╝ ╚══════╝╚══════╝ ╚═════╝   ╚═╝   ╚══════╝                                                      
+"""
+
 
 def check_server(cli_version: str, server_url: str, quiet: bool = False) -> None:
     """Runs a health check and a version check against the server."""

src/fidesctl/core/api_helpers.py

@@ -82,24 +82,32 @@ def list_server_resources(
     headers: Dict[str, str],
     resource_type: str,
     exclude_keys: List[str],
-) -> List[FidesModel]:
+    raw: bool = False,
+) -> Optional[Union[List[FidesModel], List[Dict]]]:
     """
     Get a list of resources from the server and return them as parsed objects.
 
     Returns an empty list if no resources are found or if the API returns an error.
     """
     response: Response = api.ls(url=url, resource_type=resource_type, headers=headers)
-    server_resources: List[FidesModel] = (
+    server_resources = (
         [
-            parse_dict(
-                resource_type=resource_type,
-                resource=resource,
-                from_server=True,
-            )
+            resource
             for resource in response.json()
             if isinstance(resource, dict) and resource["fides_key"] not in exclude_keys
         ]
         if response.status_code >= 200 and response.status_code <= 299
         else []
     )
+
+    if not raw and server_resources:
+        server_resources = [
+            parse_dict(
+                resource_type=resource_type,
+                resource=resource_dict,
+                from_server=True,
+            )
+            for resource_dict in server_resources
+        ]
+
     return server_resources