From 81d5e9bd850ed79ecb7a20c063e3b3ce58498633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Mon, 16 Oct 2023 23:59:21 +0700 Subject: [PATCH] Up --- lib/livebook/hubs/dockerfile.ex | 284 ++++++++++++++++++ .../components/core_components.ex | 17 +- lib/livebook_web/live/app_helpers.ex | 207 ------------- .../live/hub/edit/team_component.ex | 14 +- .../live/session_live/app_docker_component.ex | 101 ++----- .../hubs/dockerfile_test.exs} | 131 ++++++-- 6 files changed, 437 insertions(+), 317 deletions(-) create mode 100644 lib/livebook/hubs/dockerfile.ex rename test/{livebook_web/live/app_helpers_test.exs => livebook/hubs/dockerfile_test.exs} (59%) diff --git a/lib/livebook/hubs/dockerfile.ex b/lib/livebook/hubs/dockerfile.ex new file mode 100644 index 00000000000..18c544c088a --- /dev/null +++ b/lib/livebook/hubs/dockerfile.ex @@ -0,0 +1,284 @@ +defmodule Livebook.Hubs.Dockerfile do + # This module is responsible for building Dockerfile to deploy apps. + + import Ecto.Changeset + + alias Livebook.Hubs + + @type config :: %{ + deploy_all: boolean(), + docker_tag: String.t(), + zta_provider: atom() | nil, + zta_key: String.t() | nil + } + + @doc """ + Builds a changeset for app Dockerfile configuration. + """ + @spec config_changeset(map()) :: Ecto.Changeset.t() + def config_changeset(attrs \\ %{}) do + default_image = Livebook.Config.docker_images() |> hd() + + data = %{deploy_all: false, docker_tag: default_image.tag, zta_provider: nil, zta_key: nil} + + zta_types = + for provider <- Livebook.Config.identity_providers(), + not provider.read_only, + do: provider.type + + types = %{ + deploy_all: :boolean, + docker_tag: :string, + zta_provider: Ecto.ParameterizedType.init(Ecto.Enum, values: zta_types), + zta_key: :string + } + + cast({data, types}, attrs, [:deploy_all, :docker_tag, :zta_provider, :zta_key]) + |> validate_required([:deploy_all, :docker_tag]) + end + + @doc """ + Builds Dockerfile definition for app deployment. + """ + @spec build_dockerfile( + config(), + Hubs.Provider.t(), + list(Livebook.Secrets.Secret.t()), + list(Livebook.FileSystem.t()), + Livebook.FileSystem.File.t() | nil, + list(Livebook.Notebook.file_entry()), + list(Livebook.Session.Data.secrets()) + ) :: String.t() + def build_dockerfile(config, hub, hub_secrets, hub_file_systems, file, file_entries, secrets) do + base_image = Enum.find(Livebook.Config.docker_images(), &(&1.tag == config.docker_tag)) + + image = """ + FROM ghcr.io/livebook-dev/livebook:#{base_image.tag} + """ + + image_envs = format_envs(base_image.env) + + hub_type = Hubs.Provider.type(hub) + used_secrets = used_secrets(config, hub, secrets, hub_secrets) |> Enum.sort_by(& &1.name) + hub_config = format_hub_config(hub_type, config, hub, hub_file_systems, used_secrets) + + apps_config = """ + # Apps configuration + ENV LIVEBOOK_APPS_PATH "/apps" + ENV LIVEBOOK_APPS_PATH_WARMUP "manual" + ENV LIVEBOOK_APPS_PATH_HUB_ID "#{hub.id}" + """ + + notebook = + if config.deploy_all do + """ + # Notebooks and files + COPY . /apps + """ + else + notebook_file_name = Livebook.FileSystem.File.name(file) + + notebook = + """ + # Notebook + COPY #{notebook_file_name} /apps/ + """ + + attachments = + file_entries + |> Enum.filter(&(&1.type == :attachment)) + |> Enum.sort_by(& &1.name) + + if attachments == [] do + notebook + else + list = Enum.map_join(attachments, " ", &"files/#{&1.name}") + + """ + # Files + COPY #{list} /apps/files/ + + #{notebook}\ + """ + end + end + + apps_warmup = """ + # Cache apps setup at build time + RUN /app/bin/warmup_apps.sh + """ + + [ + image, + image_envs, + hub_config, + apps_config, + notebook, + apps_warmup + ] + |> Enum.reject(&is_nil/1) + |> Enum.join("\n") + end + + defp format_hub_config("team", config, hub, hub_file_systems, used_secrets) do + base_env = + """ + ARG TEAMS_KEY="#{hub.teams_key}" + + # Teams Hub configuration for airgapped deployment + ENV LIVEBOOK_TEAMS_KEY ${TEAMS_KEY} + ENV LIVEBOOK_TEAMS_NAME "#{hub.hub_name}" + ENV LIVEBOOK_TEAMS_OFFLINE_KEY "#{hub.org_public_key}" + """ + + secrets = + if used_secrets != [] do + """ + ENV LIVEBOOK_TEAMS_SECRETS "#{encrypt_secrets_to_dockerfile(used_secrets, hub)}" + """ + end + + file_systems = + if hub_file_systems != [] do + """ + ENV LIVEBOOK_TEAMS_FS "#{encrypt_file_systems_to_dockerfile(hub_file_systems, hub)}" + """ + end + + zta = + if zta_configured?(config) do + """ + ENV LIVEBOOK_IDENTITY_PROVIDER "#{config.zta_provider}:#{config.zta_key}" + """ + end + + [base_env, secrets, file_systems, zta] + |> Enum.reject(&is_nil/1) + |> Enum.join() + end + + defp format_hub_config("personal", _config, _hub, _hub_file_systems, used_secrets) do + if used_secrets != [] do + envs = used_secrets |> Enum.map(&{"LB_" <> &1.name, &1.value}) |> format_envs() + + """ + # Personal Hub secrets + #{envs}\ + """ + end + end + + defp format_envs([]), do: nil + + defp format_envs(list) do + Enum.map_join(list, fn {key, value} -> ~s/ENV #{key} "#{value}"\n/ end) + end + + defp encrypt_secrets_to_dockerfile(secrets, hub) do + secrets_map = + for %{name: name, value: value} <- secrets, + into: %{}, + do: {name, value} + + encrypt_to_dockerfile(hub, secrets_map) + end + + defp encrypt_file_systems_to_dockerfile(file_systems, hub) do + file_systems = + for file_system <- file_systems do + file_system + |> Livebook.FileSystem.dump() + |> Map.put_new(:type, Livebook.FileSystems.type(file_system)) + end + + encrypt_to_dockerfile(hub, file_systems) + end + + defp encrypt_to_dockerfile(hub, data) do + secret_key = Livebook.Teams.derive_key(hub.teams_key) + + data + |> Jason.encode!() + |> Livebook.Teams.encrypt(secret_key) + end + + defp used_secrets(config, hub, secrets, hub_secrets) do + if config.deploy_all do + hub_secrets + else + for {_, secret} <- secrets, secret.hub_id == hub.id, do: secret + end + end + + defp zta_configured?(config) do + config.zta_provider != nil and config.zta_key != nil + end + + @doc """ + Returns a list of Dockerfile-related warnings. + + The returned messages may include HTML. + """ + @spec warnings( + config(), + Hubs.Provider.t(), + list(Livebook.Secrets.Secret.t()), + Livebook.Notebook.AppSettings.t(), + list(Livebook.Notebook.file_entry()), + list(Livebook.Session.Data.secrets()) + ) :: list(String.t()) + def warnings(config, hub, hub_secrets, app_settings, file_entries, secrets) do + common_warnings = + [ + if Livebook.Session.Data.session_secrets(secrets, hub.id) != [] do + "The notebook uses session secrets, but those are not available to deployed apps." <> + " Convert them to Hub secrets instead." + end + ] + + hub_warnings = + case Hubs.Provider.type(hub) do + "personal" -> + [ + if used_secrets(config, hub, secrets, hub_secrets) != [] do + "You are deploying an app with secrets and the secrets are included in the Dockerfile" <> + " as environment variables. If someone else deploys this app, they must also set the" <> + " same secrets. Use Livebook Teams to automatically encrypt and synchronize secrets" <> + " across your team and deployments." + end, + if module = find_hub_file_system(file_entries) do + name = LivebookWeb.FileSystemHelpers.file_system_name(module) + + "The #{name} file storage, defined in your personal hub, will not be available in the Docker image." <> + " You must either download all references as attachments or use Livebook Teams to automatically" <> + " encrypt and synchronize file storages across your team and deployments." + end, + if app_settings.access_type == :public do + teams_link = + ~s{Livebook Teams} + + "This app has no password configuration and anyone with access to the server will be able" <> + " to use it. You may either configure a password or use #{teams_link} to add Zero Trust Authentication" <> + " to your deployed notebooks." + end + ] + + "team" -> + [ + if app_settings.access_type == :public and not zta_configured?(config) do + "This app has no password configuration and anyone with access to the server will be able" <> + " to use it. You may either configure a password or configure Zero Trust Authentication." + end + ] + end + + Enum.reject(common_warnings ++ hub_warnings, &is_nil/1) + end + + defp find_hub_file_system(file_entries) do + Enum.find_value(file_entries, fn entry -> + entry.type == :file && entry.file.file_system_module != FileSystem.Local && + entry.file.file_system_module + end) + end +end diff --git a/lib/livebook_web/components/core_components.ex b/lib/livebook_web/components/core_components.ex index 0838633213c..293e76e4f6b 100644 --- a/lib/livebook_web/components/core_components.ex +++ b/lib/livebook_web/components/core_components.ex @@ -94,11 +94,17 @@ defmodule LivebookWeb.CoreComponents do <.message_box kind={:info} message="🦊 in a 📦" /> + <.message_box kind={:info}> + 🦊 in a 📦 + + """ - attr :message, :string, required: true + attr :message, :string, default: nil attr :kind, :atom, values: [:info, :success, :warning, :error] + slot :inner_block + def message_box(assigns) do ~H"""
-
<%= @message %>
+
<%= @message %>
+
+ <%= render_slot(@inner_block) %> +
""" end diff --git a/lib/livebook_web/live/app_helpers.ex b/lib/livebook_web/live/app_helpers.ex index 08c0812322a..fb83fc07821 100644 --- a/lib/livebook_web/live/app_helpers.ex +++ b/lib/livebook_web/live/app_helpers.ex @@ -1,8 +1,6 @@ defmodule LivebookWeb.AppHelpers do use LivebookWeb, :html - import Ecto.Changeset - alias Livebook.Hubs @doc """ @@ -84,31 +82,6 @@ defmodule LivebookWeb.AppHelpers do ) end - @doc """ - Builds a changeset for app Dockerfile configuration. - """ - @spec docker_config_changeset(map()) :: Ecto.Changeset.t() - def docker_config_changeset(attrs \\ %{}) do - default_image = Livebook.Config.docker_images() |> hd() - - data = %{deploy_all: false, docker_tag: default_image.tag, zta_provider: nil, zta_key: nil} - - zta_types = - for provider <- Livebook.Config.identity_providers(), - not provider.read_only, - do: provider.type - - types = %{ - deploy_all: :boolean, - docker_tag: :string, - zta_provider: Ecto.ParameterizedType.init(Ecto.Enum, values: zta_types), - zta_key: :string - } - - cast({data, types}, attrs, [:deploy_all, :docker_tag, :zta_provider, :zta_key]) - |> validate_required([:deploy_all, :docker_tag]) - end - @doc """ Renders form fields for Dockerfile configuration. """ @@ -252,184 +225,4 @@ defmodule LivebookWeb.AppHelpers do defp zta_metadata(zta_provider) do Enum.find(Livebook.Config.identity_providers(), &(&1.type == zta_provider)) end - - @doc """ - Builds Dockerfile definition for app deployment. - """ - @spec build_dockerfile( - map(), - Hubs.Provider.t(), - list(Livebook.Secrets.Secret.t()), - list(Livebook.FileSystem.t()), - Livebook.FileSystem.File.t() | nil, - list(Livebook.Notebook.file_entry()), - list(Livebook.Session.Data.secrets()) - ) :: String.t() - def build_dockerfile(config, hub, hub_secrets, hub_file_systems, file, file_entries, secrets) do - base_image = Enum.find(Livebook.Config.docker_images(), &(&1.tag == config.docker_tag)) - - image = """ - FROM ghcr.io/livebook-dev/livebook:#{base_image.tag} - """ - - image_envs = format_envs(base_image.env) - - hub_config = - case Hubs.Provider.type(hub) do - "team" -> - format_team_hub_config(config, hub, hub_secrets, hub_file_systems) - - "personal" -> - format_personal_hub_config(config, hub, hub_secrets, secrets) - end - - apps_config = """ - # Apps configuration - ENV LIVEBOOK_APPS_PATH "/apps" - ENV LIVEBOOK_APPS_PATH_WARMUP "manual" - ENV LIVEBOOK_APPS_PATH_HUB_ID "#{hub.id}" - """ - - notebook = - if config.deploy_all do - """ - # Notebooks and files - COPY . /apps - """ - else - notebook_file_name = Livebook.FileSystem.File.name(file) - - notebook = - """ - # Notebook - COPY #{notebook_file_name} /apps/ - """ - - attachments = - file_entries - |> Enum.filter(&(&1.type == :attachment)) - |> Enum.sort_by(& &1.name) - - if attachments == [] do - notebook - else - list = Enum.map_join(attachments, " ", &"files/#{&1.name}") - - """ - # Files - COPY #{list} /apps/files/ - - #{notebook}\ - """ - end - end - - apps_warmup = """ - # Cache apps setup at build time - RUN /app/bin/warmup_apps.sh - """ - - [ - image, - image_envs, - hub_config, - apps_config, - notebook, - apps_warmup - ] - |> Enum.reject(&is_nil/1) - |> Enum.join("\n") - end - - defp format_team_hub_config(config, hub, secrets, file_systems) do - base_env = - """ - ARG TEAMS_KEY="#{hub.teams_key}" - - # Teams Hub configuration for airgapped deployment - ENV LIVEBOOK_TEAMS_KEY ${TEAMS_KEY} - ENV LIVEBOOK_TEAMS_NAME "#{hub.hub_name}" - ENV LIVEBOOK_TEAMS_OFFLINE_KEY "#{hub.org_public_key}" - """ - - secrets = - if secrets != [] do - """ - ENV LIVEBOOK_TEAMS_SECRETS "#{encrypt_secrets_to_dockerfile(secrets, hub)}" - """ - end - - file_systems = - if file_systems != [] do - """ - ENV LIVEBOOK_TEAMS_FS "#{encrypt_file_systems_to_dockerfile(file_systems, hub)}" - """ - end - - zta = - if config.zta_provider && config.zta_key do - """ - ENV LIVEBOOK_IDENTITY_PROVIDER "#{config.zta_provider}:#{config.zta_key}" - """ - end - - [base_env, secrets, file_systems, zta] - |> Enum.reject(&is_nil/1) - |> Enum.join() - end - - defp format_personal_hub_config(config, hub, hub_secrets, secrets) do - secrets = used_secrets(config, hub, secrets, hub_secrets) |> Enum.sort_by(& &1.name) - - if secrets != [] do - envs = secrets |> Enum.map(&{"LB_" <> &1.name, &1.value}) |> format_envs() - - """ - # Personal Hub secrets - #{envs}\ - """ - end - end - - defp format_envs([]), do: nil - - defp format_envs(list) do - Enum.map_join(list, fn {key, value} -> ~s/ENV #{key} "#{value}"\n/ end) - end - - defp encrypt_secrets_to_dockerfile(secrets, hub) do - secrets_map = - for %{name: name, value: value} <- secrets, - into: %{}, - do: {name, value} - - encrypt_to_dockerfile(hub, secrets_map) - end - - defp encrypt_file_systems_to_dockerfile(file_systems, hub) do - file_systems = - for file_system <- file_systems do - file_system - |> Livebook.FileSystem.dump() - |> Map.put_new(:type, Livebook.FileSystems.type(file_system)) - end - - encrypt_to_dockerfile(hub, file_systems) - end - - defp encrypt_to_dockerfile(hub, data) do - secret_key = Livebook.Teams.derive_key(hub.teams_key) - - data - |> Jason.encode!() - |> Livebook.Teams.encrypt(secret_key) - end - - defp used_secrets(config, hub, secrets, hub_secrets) do - if config.deploy_all do - hub_secrets - else - for {_, secret} <- secrets, secret.hub_id == hub.id, do: secret - end - end end diff --git a/lib/livebook_web/live/hub/edit/team_component.ex b/lib/livebook_web/live/hub/edit/team_component.ex index 3eb1828e0a9..d4384cd3025 100644 --- a/lib/livebook_web/live/hub/edit/team_component.ex +++ b/lib/livebook_web/live/hub/edit/team_component.ex @@ -43,8 +43,8 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do hub_metadata: Provider.to_metadata(assigns.hub), is_default: is_default? ) - |> assign_new(:docker_config_changeset, fn -> - LivebookWeb.AppHelpers.docker_config_changeset() + |> assign_new(:config_changeset, fn -> + Livebook.Hubs.Dockerfile.config_changeset() end) |> update_dockerfile() |> assign_form(changeset)} @@ -210,7 +210,7 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do <.form :let={f} - for={@docker_config_changeset} + for={@config_changeset} as={:data} phx-change="validate_dockerfile" phx-target={@myself} @@ -416,12 +416,12 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do def handle_event("validate_dockerfile", %{"data" => data}, socket) do changeset = data - |> LivebookWeb.AppHelpers.docker_config_changeset() + |> Livebook.Hubs.Dockerfile.config_changeset() |> Map.replace!(:action, :validate) {:noreply, socket - |> assign(docker_config_changeset: changeset) + |> assign(config_changeset: changeset) |> update_dockerfile()} end @@ -445,14 +445,14 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do defp update_dockerfile(socket) do config = - socket.assigns.docker_config_changeset + socket.assigns.config_changeset |> Ecto.Changeset.apply_changes() |> Map.replace!(:deploy_all, true) %{hub: hub, secrets: hub_secrets, file_systems: hub_file_systems} = socket.assigns dockerfile = - LivebookWeb.AppHelpers.build_dockerfile( + Livebook.Hubs.Dockerfile.build_dockerfile( config, hub, hub_secrets, diff --git a/lib/livebook_web/live/session_live/app_docker_component.ex b/lib/livebook_web/live/session_live/app_docker_component.ex index 77b58efe326..6f0e5b8b01e 100644 --- a/lib/livebook_web/live/session_live/app_docker_component.ex +++ b/lib/livebook_web/live/session_live/app_docker_component.ex @@ -17,7 +17,7 @@ defmodule LivebookWeb.SessionLive.AppDockerComponent do hub_secrets: Hubs.get_secrets(assigns.hub), hub_file_systems: Hubs.get_file_systems(assigns.hub, hub_only: true) ) - |> assign_new(:changeset, fn -> LivebookWeb.AppHelpers.docker_config_changeset() end) + |> assign_new(:changeset, fn -> Livebook.Hubs.Dockerfile.config_changeset() end) |> assign_new(:save_result, fn -> nil end) |> update_dockerfile()} end @@ -34,13 +34,13 @@ defmodule LivebookWeb.SessionLive.AppDockerComponent do settings_valid?={@settings_valid?} hub={@hub} hub_secrets={@hub_secrets} - app_settings={@app_settings} hub_file_systems={@hub_file_systems} file_entries={@file_entries} secrets={@secrets} changeset={@changeset} session={@session} dockerfile={@dockerfile} + warnings={@warnings} save_result={@save_result} myself={@myself} /> @@ -83,16 +83,6 @@ defmodule LivebookWeb.SessionLive.AppDockerComponent do You can deploy this app in the cloud using Docker. To do that, configure the deployment and then use the generated Dockerfile.

-
- <.message_box - :for={ - warning <- - warnings(@changeset, @hub, @hub_secrets, @app_settings, @file_entries, @secrets) - } - kind={:warning} - message={warning} - /> -

<.label>Hub @@ -100,6 +90,11 @@ defmodule LivebookWeb.SessionLive.AppDockerComponent do <%= @hub.hub_name %>

+
+ <.message_box :for={warning <- @warnings} kind={:warning}> + <%= raw(warning) %> + +
<.form :let={f} for={@changeset} as={:data} phx-change="validate" phx-target={@myself}> @@ -143,7 +138,7 @@ defmodule LivebookWeb.SessionLive.AppDockerComponent do def handle_event("validate", %{"data" => data}, socket) do changeset = data - |> LivebookWeb.AppHelpers.docker_config_changeset() + |> Livebook.Hubs.Dockerfile.config_changeset() |> Map.replace!(:action, :validate) {:noreply, assign(socket, changeset: changeset) |> update_dockerfile()} @@ -162,7 +157,7 @@ defmodule LivebookWeb.SessionLive.AppDockerComponent do end defp update_dockerfile(socket) when socket.assigns.file == nil do - assign(socket, dockerfile: nil) + assign(socket, dockerfile: nil, warnings: []) end defp update_dockerfile(socket) do @@ -174,11 +169,12 @@ defmodule LivebookWeb.SessionLive.AppDockerComponent do hub_file_systems: hub_file_systems, file: file, file_entries: file_entries, - secrets: secrets + secrets: secrets, + app_settings: app_settings } = socket.assigns dockerfile = - LivebookWeb.AppHelpers.build_dockerfile( + Livebook.Hubs.Dockerfile.build_dockerfile( config, hub, hub_secrets, @@ -188,69 +184,16 @@ defmodule LivebookWeb.SessionLive.AppDockerComponent do secrets ) - assign(socket, :dockerfile, dockerfile) - end - - defp warnings(changeset, hub, hub_secrets, app_settings, file_entries, secrets) do - config = apply_changes(changeset) - - common_warnings = - [ - if Livebook.Session.Data.session_secrets(secrets, hub.id) != [] do - "The notebook uses session secrets, but those are not available to deployed apps." <> - " Convert them to Hub secrets instead." - end - ] - - hub_warnings = - case Hubs.Provider.type(hub) do - "personal" -> - [ - if used_secrets(config, hub, secrets, hub_secrets) != [] do - "You are deploying an app with secrets and the secrets are included in the Dockerfile" <> - " as environment variables. If someone else deploys this app, they must also set the" <> - " same secrets. Use Livebook Teams to automatically encrypt and synchronize secrets" <> - " across your team and deployments." - end, - if module = find_hub_file_system(file_entries) do - name = LivebookWeb.FileSystemHelpers.file_system_name(module) - - "The #{name} file storage, defined in your personal hub, will not be available in the Docker image." <> - " You must either download all references as attachments or use Livebook Teams to automatically" <> - " encrypt and synchronize file storages across your team and deployments." - end, - if app_settings.access_type == :public do - "This app has no password configuration and anyone with access to the server will be able" <> - " to use it. Either configure a password or use Livebook Teams to add Zero Trust Authentication" <> - " to your deployed notebooks." - end - ] - - "team" -> - [ - if app_settings.access_type == :public and - (config.zta_provider == nil or config.zta_key == nil) do - "This app has no password configuration and anyone with access to the server will be able" <> - " to use it. Either configure a password or configure Zero Trust Authentication." - end - ] - end - - Enum.reject(common_warnings ++ hub_warnings, &is_nil/1) - end - - defp find_hub_file_system(file_entries) do - Enum.find_value(file_entries, fn entry -> - entry.type == :file && entry.file.file_system_module != FileSystem.Local && - entry.file.file_system_module - end) - end + warnings = + Livebook.Hubs.Dockerfile.warnings( + config, + hub, + hub_secrets, + app_settings, + file_entries, + secrets + ) - defp used_secrets(config, hub, secrets, hub_secrets) do - if config.deploy_all do - hub_secrets - else - for {_, secret} <- secrets, secret.hub_id == hub.id, do: secret - end + assign(socket, dockerfile: dockerfile, warnings: warnings) end end diff --git a/test/livebook_web/live/app_helpers_test.exs b/test/livebook/hubs/dockerfile_test.exs similarity index 59% rename from test/livebook_web/live/app_helpers_test.exs rename to test/livebook/hubs/dockerfile_test.exs index 1d9f343801c..b9cdca44d60 100644 --- a/test/livebook_web/live/app_helpers_test.exs +++ b/test/livebook/hubs/dockerfile_test.exs @@ -1,9 +1,9 @@ -defmodule LivebookWeb.AppHelpersTest do +defmodule Livebook.Hubs.DockerfileTest do use ExUnit.Case, async: true import Livebook.TestHelpers - alias LivebookWeb.AppHelpers + alias Livebook.Hubs.Dockerfile alias Livebook.Hubs alias Livebook.Secrets.Secret @@ -11,13 +11,13 @@ defmodule LivebookWeb.AppHelpersTest do test "deploying a single notebook in personal hub" do config = %{} - |> AppHelpers.docker_config_changeset() + |> Dockerfile.config_changeset() |> Ecto.Changeset.apply_changes() hub = Hubs.fetch_hub!(Hubs.Personal.id()) file = Livebook.FileSystem.File.local(p("/notebook.livemd")) - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, [], %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile == """ FROM ghcr.io/livebook-dev/livebook:latest @@ -38,10 +38,10 @@ defmodule LivebookWeb.AppHelpersTest do config = %{docker_tag: "latest-cuda11.8"} - |> AppHelpers.docker_config_changeset() + |> Dockerfile.config_changeset() |> Ecto.Changeset.apply_changes() - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, [], %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile =~ """ FROM ghcr.io/livebook-dev/livebook:latest-cuda11.8 @@ -56,7 +56,7 @@ defmodule LivebookWeb.AppHelpersTest do %{type: :attachment, name: "data.csv"} ] - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, file_entries, %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, file_entries, %{}) assert dockerfile =~ """ @@ -72,7 +72,7 @@ defmodule LivebookWeb.AppHelpersTest do hub_secrets = [secret] secrets = %{"TEST" => secret, "SESSION" => session_secret} - dockerfile = AppHelpers.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) + dockerfile = Dockerfile.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) assert dockerfile =~ """ @@ -86,13 +86,13 @@ defmodule LivebookWeb.AppHelpersTest do test "deploying a directory in personal hub" do config = %{deploy_all: true} - |> AppHelpers.docker_config_changeset() + |> Dockerfile.config_changeset() |> Ecto.Changeset.apply_changes() hub = Hubs.fetch_hub!(Hubs.Personal.id()) file = Livebook.FileSystem.File.local(p("/notebook.livemd")) - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, [], %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile =~ """ # Notebooks and files @@ -108,7 +108,7 @@ defmodule LivebookWeb.AppHelpersTest do hub_secrets = [secret, unused_secret] secrets = %{"TEST" => secret, "SESSION" => session_secret} - dockerfile = AppHelpers.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) + dockerfile = Dockerfile.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) assert dockerfile =~ """ @@ -123,13 +123,13 @@ defmodule LivebookWeb.AppHelpersTest do test "deploying a single notebook in teams hub" do config = %{} - |> AppHelpers.docker_config_changeset() + |> Dockerfile.config_changeset() |> Ecto.Changeset.apply_changes() hub = team_hub() file = Livebook.FileSystem.File.local(p("/notebook.livemd")) - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, [], %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile == """ FROM ghcr.io/livebook-dev/livebook:latest @@ -157,10 +157,10 @@ defmodule LivebookWeb.AppHelpersTest do config = %{docker_tag: "latest-cuda11.8"} - |> AppHelpers.docker_config_changeset() + |> Dockerfile.config_changeset() |> Ecto.Changeset.apply_changes() - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, [], %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile =~ """ FROM ghcr.io/livebook-dev/livebook:latest-cuda11.8 @@ -175,7 +175,7 @@ defmodule LivebookWeb.AppHelpersTest do %{type: :attachment, name: "data.csv"} ] - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, file_entries, %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, file_entries, %{}) assert dockerfile =~ """ @@ -191,7 +191,7 @@ defmodule LivebookWeb.AppHelpersTest do hub_secrets = [secret] secrets = %{"TEST" => secret, "SESSION" => session_secret} - dockerfile = AppHelpers.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) + dockerfile = Dockerfile.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) assert dockerfile =~ "ENV LIVEBOOK_TEAMS_SECRETS" refute dockerfile =~ "ENV TEST" @@ -202,7 +202,7 @@ defmodule LivebookWeb.AppHelpersTest do file_system = Livebook.Factory.build(:fs_s3) file_systems = [file_system] - dockerfile = AppHelpers.build_dockerfile(config, hub, [], file_systems, file, [], %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], file_systems, file, [], %{}) assert dockerfile =~ "ENV LIVEBOOK_TEAMS_FS" @@ -210,10 +210,10 @@ defmodule LivebookWeb.AppHelpersTest do config = %{zta_provider: :cloudflare, zta_key: "cloudflare_key"} - |> AppHelpers.docker_config_changeset() + |> Dockerfile.config_changeset() |> Ecto.Changeset.apply_changes() - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, [], %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile =~ ~S/ENV LIVEBOOK_IDENTITY_PROVIDER "cloudflare:cloudflare_key"/ end @@ -221,13 +221,13 @@ defmodule LivebookWeb.AppHelpersTest do test "deploying a directory in teams hub" do config = %{deploy_all: true} - |> AppHelpers.docker_config_changeset() + |> Dockerfile.config_changeset() |> Ecto.Changeset.apply_changes() hub = team_hub() file = Livebook.FileSystem.File.local(p("/notebook.livemd")) - dockerfile = AppHelpers.build_dockerfile(config, hub, [], [], file, [], %{}) + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile =~ """ # Notebooks and files @@ -236,6 +236,93 @@ defmodule LivebookWeb.AppHelpersTest do end end + describe "warnings/6" do + test "warns when session secrets are used" do + config = + %{} + |> Dockerfile.config_changeset() + |> Ecto.Changeset.apply_changes() + + hub = Hubs.fetch_hub!(Hubs.Personal.id()) + app_settings = Livebook.Notebook.AppSettings.new() + + session_secret = %Secret{name: "SESSION", value: "test", hub_id: nil} + secrets = %{"SESSION" => session_secret} + + assert [warning] = Dockerfile.warnings(config, hub, [], app_settings, [], secrets) + assert warning =~ "The notebook uses session secrets" + end + + test "warns when hub secrets are used from personal hub" do + config = + %{} + |> Dockerfile.config_changeset() + |> Ecto.Changeset.apply_changes() + + hub = Hubs.fetch_hub!(Hubs.Personal.id()) + app_settings = Livebook.Notebook.AppSettings.new() + + secret = %Secret{name: "TEST", value: "test", hub_id: hub.id} + + hub_secrets = [secret] + secrets = %{"TEST" => secret} + + assert [warning] = Dockerfile.warnings(config, hub, hub_secrets, app_settings, [], secrets) + assert warning =~ "secrets are included in the Dockerfile" + end + + test "warns when there is a reference to external file system from personal hub" do + config = + %{} + |> Dockerfile.config_changeset() + |> Ecto.Changeset.apply_changes() + + hub = Hubs.fetch_hub!(Hubs.Personal.id()) + app_settings = Livebook.Notebook.AppSettings.new() + + file_system = Livebook.Factory.build(:fs_s3) + + file_entries = [ + %{type: :file, file: Livebook.FileSystem.File.new(file_system, "/data.csv")} + ] + + assert [warning] = Dockerfile.warnings(config, hub, [], app_settings, file_entries, %{}) + + assert warning =~ + "The S3 file storage, defined in your personal hub, will not be available in the Docker image" + end + + test "warns when the app has no password in personal hub" do + config = + %{} + |> Dockerfile.config_changeset() + |> Ecto.Changeset.apply_changes() + + hub = Hubs.fetch_hub!(Hubs.Personal.id()) + app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public} + + assert [warning] = Dockerfile.warnings(config, hub, [], app_settings, [], %{}) + assert warning =~ "This app has no password configuration" + end + + test "warns when the app has no password and no ZTA in teams hub" do + config = + %{} + |> Dockerfile.config_changeset() + |> Ecto.Changeset.apply_changes() + + hub = team_hub() + app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public} + + assert [warning] = Dockerfile.warnings(config, hub, [], app_settings, [], %{}) + assert warning =~ "This app has no password configuration" + + config = %{config | zta_provider: :cloudflare, zta_key: "key"} + + assert [] = Dockerfile.warnings(config, hub, [], app_settings, [], %{}) + end + end + defp team_hub() do Livebook.Factory.build(:team, id: "team-org-name-387",