v1.4.3

pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
   "pytest",
   # Linting
   "ruff",
-  "phidata==1.4.1",
+  "phidata==1.4.3",
   # Dataframe libraries
   "pandas",
   "polars",

requirements.txt

@@ -38,8 +38,8 @@ oauthlib==3.2.2
 packaging==23.0
 pandas==1.5.3
 pathspec==0.9.0
-phidata==1.4.1
-phiterm==1.4.1
+phidata==1.4.3
+phiterm==1.4.3
 platformdirs==2.6.2
 pluggy==1.0.0
 polars==0.16.1

workflows/notebooks/notebooks_daily.py

@@ -27,7 +27,6 @@
     tags=["notebooks"],
     catchup=False,
 ) as dag:
-
     hello_world = PapermillOperator(
         task_id="hello_world",
         input_nb=str(notebooks_dir.joinpath("examples", "hello_world.ipynb")),

workspace/dev/airflow/docker_resources.py

@@ -41,10 +41,12 @@
 # Mount the ws_repo using a docker volume
 mount_workspace: bool = True
 # Read env variables from env/dev_airflow_env.yml
-dev_airflow_env_file: Path = ws_settings.ws_dir.joinpath("env/dev_airflow_env.yml")
+dev_airflow_env_file: Path = ws_settings.ws_root.joinpath(
+    "workspace/env/dev_airflow_env.yml"
+)
 # Read secrets from secrets/dev_airflow_secrets.yml
-dev_airflow_secrets_file: Path = ws_settings.ws_dir.joinpath(
-    "secrets/dev_airflow_secrets.yml"
+dev_airflow_secrets_file: Path = ws_settings.ws_root.joinpath(
+    "workspace/secrets/dev_airflow_secrets.yml"
 )
 # Add airflow configuration using env variables
 dev_airflow_env: Dict[str, str] = {
@@ -63,7 +65,7 @@
     name=f"{ws_settings.image_repo}/airflow-{ws_settings.image_suffix}",
     tag=ws_settings.dev_env,
     enabled=ws_settings.build_images,
-    path=str(ws_settings.ws_dir.parent),
+    path=str(ws_settings.ws_root),
     dockerfile="workspace/dev/images/airflow.Dockerfile",
     pull=ws_settings.force_pull_images,
     push_image=ws_settings.push_images,

workspace/dev/airflow/resources/webserver_config.py

@@ -125,13 +125,11 @@ def get_roles_for_gh_team(team_payload: List[dict]) -> List[str]:
 
 
 class OauthAuthorizer(AirflowSecurityManager):
-
     # For other providers:
     # https://github.com/dpgaspar/Flask-AppBuilder/blob/master/flask_appbuilder/security/manager.py#L550
     def get_oauth_user_info(
         self, provider: str, resp: Any
     ) -> dict[str, Union[str, list[str]]]:
-
         logging.info(f"Getting user info from {provider}")
 
         if provider == "google":

workspace/dev/databox.py

@@ -18,7 +18,7 @@
     name=f"{ws_settings.image_repo}/databox-{ws_settings.image_suffix}",
     tag=ws_settings.dev_env,
     enabled=ws_settings.build_images,
-    path=str(ws_settings.ws_dir.parent),
+    path=str(ws_settings.ws_root),
     dockerfile="workspace/dev/images/databox.Dockerfile",
     pull=ws_settings.force_pull_images,
     push_image=ws_settings.push_images,

workspace/dev/jupyter/docker_resources.py

@@ -13,7 +13,7 @@
     name=f"{ws_settings.image_repo}/jupyter-{ws_settings.image_suffix}",
     tag=ws_settings.dev_env,
     enabled=ws_settings.build_images,
-    path=str(ws_settings.ws_dir.parent),
+    path=str(ws_settings.ws_root),
     dockerfile="workspace/dev/images/jupyter.Dockerfile",
     pull=ws_settings.force_pull_images,
     push_image=ws_settings.push_images,
@@ -28,9 +28,11 @@
     # The jupyter_lab_config is mounted when creating the image
     jupyter_config_file="/usr/local/jupyter/jupyter_lab_config.py",
     # Read env variables from env/dev_jupyter_env.yml
-    env_file=ws_settings.ws_dir.joinpath("env/dev_jupyter_env.yml"),
+    env_file=ws_settings.ws_root.joinpath("workspace/env/dev_jupyter_env.yml"),
     # Read secrets from secrets/dev_jupyter_secrets.yml
-    secrets_file=ws_settings.ws_dir.joinpath("secrets/dev_jupyter_secrets.yml"),
+    secrets_file=ws_settings.ws_root.joinpath(
+        "workspace/secrets/dev_jupyter_secrets.yml"
+    ),
     use_cache=ws_settings.use_cache,
     # Run the notebook server on jupyter.dp
     container_labels={

workspace/dev/superset/docker_resources.py

@@ -38,18 +38,20 @@
 mount_resources: bool = True
 dev_superset_resources: str = "workspace/dev/superset/resources"
 # Read env variables from env/dev_superset_env.yml
-dev_superset_env_file: Path = ws_settings.ws_dir.joinpath("env/dev_superset_env.yml")
+dev_superset_env_file: Path = ws_settings.ws_root.joinpath(
+    "workspace/env/dev_superset_env.yml"
+)
 # Read secrets from secrets/dev_superset_secrets.yml
-dev_superset_secrets_file: Path = ws_settings.ws_dir.joinpath(
-    "secrets/dev_superset_secrets.yml"
+dev_superset_secrets_file: Path = ws_settings.ws_root.joinpath(
+    "workspace/secrets/dev_superset_secrets.yml"
 )
 
 # -*- Superset image
 dev_superset_image = DockerImage(
     name=f"{ws_settings.image_repo}/superset-{ws_settings.image_suffix}",
     tag=ws_settings.dev_env,
     enabled=ws_settings.build_images,
-    path=str(ws_settings.ws_dir.parent),
+    path=str(ws_settings.ws_root),
     dockerfile="workspace/dev/images/superset.Dockerfile",
     pull=ws_settings.force_pull_images,
     push_image=ws_settings.push_images,

workspace/dev/superset/resources/pythonpath_dev/custom_sso_security_manager.py

@@ -47,11 +47,9 @@ def get_roles_for_gh_team(team_payload: List[dict]) -> List[str]:
 
 
 class CustomSsoSecurityManager(SupersetSecurityManager):
-
     # For other providers:
     # https://github.com/dpgaspar/Flask-AppBuilder/blob/master/flask_appbuilder/security/manager.py#L550
     def oauth_user_info(self, provider, response=None):
-
         logging.info(f"Getting user info from {provider}")
 
         if provider == "google":

workspace/prd/airflow/aws_resources.py

@@ -94,7 +94,9 @@
     db_subnet_group=prd_rds_subnet_group,
     enable_performance_insights=True,
     vpc_security_group_ids=ws_settings.security_groups,
-    secrets_file=ws_settings.ws_dir.joinpath("secrets/prd_airflow_db_secrets.yml"),
+    secrets_file=ws_settings.ws_root.joinpath(
+        "workspace/secrets/prd_airflow_db_secrets.yml"
+    ),
     skip_create=skip_create,
     skip_delete=skip_delete,
     wait_for_creation=wait_for_create,

workspace/prd/airflow/k8s_apps.py

@@ -42,7 +42,9 @@
     enabled=(not use_rds),
     volume_type=PostgresVolumeType.AWS_EBS,
     ebs_volume=prd_airflow_db_volume,
-    secrets_file=ws_settings.ws_dir.joinpath("secrets/prd_airflow_db_secrets.yml"),
+    secrets_file=ws_settings.ws_root.joinpath(
+        "workspace/secrets/prd_airflow_db_secrets.yml"
+    ),
     pod_node_selector=services_ng_label,
 )
 
@@ -68,10 +70,12 @@
 # Create a git-sync sidecar
 create_git_sync_sidecar: bool = True
 # Read env variables from env/prd_airflow_env.yml
-prd_airflow_env_file: Path = ws_settings.ws_dir.joinpath("env/prd_airflow_env.yml")
+prd_airflow_env_file: Path = ws_settings.ws_root.joinpath(
+    "workspace/env/prd_airflow_env.yml"
+)
 # Read secrets from secrets/prd_airflow_secrets.yml
-prd_airflow_secrets_file: Path = ws_settings.ws_dir.joinpath(
-    "secrets/prd_airflow_secrets.yml"
+prd_airflow_secrets_file: Path = ws_settings.ws_root.joinpath(
+    "workspace/secrets/prd_airflow_secrets.yml"
 )
 # Add airflow configuration using env variables
 prd_airflow_env: Dict[str, str] = {

workspace/prd/airflow/resources/webserver_config.py

@@ -125,13 +125,11 @@ def get_roles_for_gh_team(team_payload: List[dict]) -> List[str]:
 
 
 class OauthAuthorizer(AirflowSecurityManager):
-
     # For other providers:
     # https://github.com/dpgaspar/Flask-AppBuilder/blob/master/flask_appbuilder/security/manager.py#L550
     def get_oauth_user_info(
         self, provider: str, resp: Any
     ) -> dict[str, Union[str, list[str]]]:
-
         logging.info(f"Getting user info from {provider}")
 
         if provider == "google":

workspace/prd/aws_resources.py

@@ -118,8 +118,8 @@
     subject_alternative_names=[f"*.{ws_settings.prd_domain}"],
     # Store the certificate ARN in the certificate_summary_file
     store_cert_summary=True,
-    certificate_summary_file=ws_settings.ws_dir.joinpath(
-        "aws", "acm", ws_settings.prd_domain
+    certificate_summary_file=ws_settings.ws_root.joinpath(
+        "workspace", "aws", "acm", ws_settings.prd_domain
     ),
     skip_create=skip_create,
     skip_delete=skip_delete,

workspace/prd/images/__init__.py

@@ -12,7 +12,7 @@
 prd_airflow_image = DockerImage(
     name=f"{ws_settings.image_repo}/airflow-{ws_settings.image_suffix}",
     tag=ws_settings.prd_env,
-    path=str(ws_settings.ws_dir.parent),
+    path=str(ws_settings.ws_root),
     dockerfile="workspace/prd/images/airflow.Dockerfile",
     pull=ws_settings.force_pull_images,
     push_image=ws_settings.push_images,
@@ -27,7 +27,7 @@
 prd_jupyter_image = DockerImage(
     name=f"{ws_settings.image_repo}/jupyter-{ws_settings.image_suffix}",
     tag=ws_settings.prd_env,
-    path=str(ws_settings.ws_dir.parent),
+    path=str(ws_settings.ws_root),
     dockerfile="workspace/prd/images/jupyter.Dockerfile",
     pull=ws_settings.force_pull_images,
     push_image=ws_settings.push_images,
@@ -42,7 +42,7 @@
 prd_superset_image = DockerImage(
     name=f"{ws_settings.image_repo}/superset-{ws_settings.image_suffix}",
     tag=ws_settings.prd_env,
-    path=str(ws_settings.ws_dir.parent),
+    path=str(ws_settings.ws_root),
     dockerfile="workspace/prd/images/superset.Dockerfile",
     pull=ws_settings.force_pull_images,
     push_image=ws_settings.push_images,
@@ -57,7 +57,7 @@
 prd_databox_image = DockerImage(
     name=f"{ws_settings.image_repo}/databox-{ws_settings.image_suffix}",
     tag=ws_settings.prd_env,
-    path=str(ws_settings.ws_dir.parent),
+    path=str(ws_settings.ws_root),
     dockerfile="workspace/prd/images/databox.Dockerfile",
     pull=ws_settings.force_pull_images,
     push_image=ws_settings.push_images,

workspace/prd/jupyter/jupyter_lab.py

@@ -57,9 +57,11 @@
     # The jupyter_lab_config is mounted when creating the image
     jupyter_config_file="/usr/local/jupyter/jupyter_lab_config.py",
     # Read env variables from env/prd_jupyter_env.yml
-    env_file=ws_settings.ws_dir.joinpath("env/prd_jupyter_env.yml"),
+    env_file=ws_settings.ws_root.joinpath("workspace/env/prd_jupyter_env.yml"),
     # Read secrets from secrets/prd_jupyter_secrets.yml
-    secrets_file=ws_settings.ws_dir.joinpath("secrets/prd_jupyter_secrets.yml"),
+    secrets_file=ws_settings.ws_root.joinpath(
+        "workspace/secrets/prd_jupyter_secrets.yml"
+    ),
     image_pull_policy=ImagePullPolicy.ALWAYS,
     use_cache=ws_settings.use_cache,
     pod_node_selector=workers_ng_label,

workspace/prd/postgres.py

@@ -46,7 +46,9 @@
     name=f"postgres-{db_id}",
     volume_type=PostgresVolumeType.AWS_EBS,
     ebs_volume=prd_postgres_volume,
-    secrets_file=ws_settings.ws_dir.joinpath("secrets/prd_postgres_secrets.yml"),
+    secrets_file=ws_settings.ws_root.joinpath(
+        "workspace/secrets/prd_postgres_secrets.yml"
+    ),
     pod_node_selector=services_ng_label,
 )
 

workspace/prd/superset/aws_resources.py

@@ -94,7 +94,9 @@
     db_subnet_group=prd_rds_subnet_group,
     enable_performance_insights=True,
     vpc_security_group_ids=ws_settings.security_groups,
-    secrets_file=ws_settings.ws_dir.joinpath("secrets/prd_superset_db_secrets.yml"),
+    secrets_file=ws_settings.ws_root.joinpath(
+        "workspace/secrets/prd_superset_db_secrets.yml"
+    ),
     skip_create=skip_create,
     skip_delete=skip_delete,
     wait_for_creation=wait_for_create,

workspace/prd/superset/k8s_apps.py

@@ -38,7 +38,9 @@
     name="ss-db",
     volume_type=PostgresVolumeType.AWS_EBS,
     ebs_volume=prd_superset_db_volume,
-    secrets_file=ws_settings.ws_dir.joinpath("secrets/prd_superset_db_secrets.yml"),
+    secrets_file=ws_settings.ws_root.joinpath(
+        "workspace/secrets/prd_superset_db_secrets.yml"
+    ),
     pod_node_selector=services_ng_label,
 )
 
@@ -61,10 +63,12 @@
 # Create a git-sync sidecar
 create_git_sync_sidecar: bool = False
 # Read env variables from env/prd_superset_env.yml
-prd_superset_env_file: Path = ws_settings.ws_dir.joinpath("env/prd_superset_env.yml")
+prd_superset_env_file: Path = ws_settings.ws_root.joinpath(
+    "workspace/env/prd_superset_env.yml"
+)
 # Read secrets from secrets/prd_superset_secrets.yml
-prd_superset_secrets_file: Path = ws_settings.ws_dir.joinpath(
-    "secrets/prd_superset_secrets.yml"
+prd_superset_secrets_file: Path = ws_settings.ws_root.joinpath(
+    "workspace/secrets/prd_superset_secrets.yml"
 )
 
 # -*- Database configuration

workspace/prd/superset/resources/pythonpath_dev/custom_sso_security_manager.py

@@ -47,11 +47,9 @@ def get_roles_for_gh_team(team_payload: List[dict]) -> List[str]:
 
 
 class CustomSsoSecurityManager(SupersetSecurityManager):
-
     # For other providers:
     # https://github.com/dpgaspar/Flask-AppBuilder/blob/master/flask_appbuilder/security/manager.py#L550
     def oauth_user_info(self, provider, response=None):
-
         logging.info(f"Getting user info from {provider}")
 
         if provider == "google":

workspace/prd/traefik.py

@@ -116,7 +116,9 @@
     # The dashboard is gated behind a user:password, which is generated using the cmd:
     #   htpasswd -nb user password
     # You can provide the "users:password" list as DASHBOARD_AUTH_USERS in the secrets_file
-    secrets_file=ws_settings.ws_dir.joinpath("secrets/prd_traefik_secrets.yml"),
+    secrets_file=ws_settings.ws_root.joinpath(
+        "workspace/secrets/prd_traefik_secrets.yml"
+    ),
     use_cache=ws_settings.use_cache,
     pod_node_selector=services_ng_label,
     topology_spread_key=topology_spread_key,

workspace/settings.py

@@ -10,8 +10,8 @@
     ws_name="dp001",
     # Workspace git repo url: used to git-sync DAGs and Charts
     ws_repo="https://github.com/phidatahq/aws-dp-template.git",
-    # Path to the workspace directory
-    ws_dir=Path(__file__).parent.resolve(),
+    # Path to the workspace root
+    ws_root=Path(__file__).parent.parent.resolve(),
     # -*- Dev settings
     dev_env="dev",
     # -*- Dev Apps