Merge KF-5945-configure-proxy-dev-branch into main

README.md

@@ -139,6 +139,102 @@ tox -e kubeflow-remote
 tox -e kubeflow-local
 ```
 
+### Run behind proxy
+
+#### Prerequisites for KServe UATs
+
+To be able to run the KServe UATs behind proxy, first you need to configure `kserve-controller`
+and `knative-serving` charms to function behind proxy.
+
+> [!NOTE]  
+> For information on how to fill out the proxy config values, see the `Running using Notebook > Prerequisites` section below.
+
+1. Set the `http-proxy`, `https-proxy`, and `no-proxy` configs in `kserve-controller` charm
+```
+juju config kserve-controller http-proxy=<proxy_address>:<proxy_port> https-proxy=<proxy_address>:<proxy_port> no-proxy=<cluster cidr>,<service cluster ip range>,127.0.0.1,localhost,<nodes internal ip(s)>/24,<cluster hostname>,.svc,.local
+```
+
+2. Set the `http-proxy`, `https-proxy`, and `no-proxy` configs in `knative-serving` charm
+```
+juju config knative-serving http-proxy=<proxy_address>:<proxy_port> https-proxy=<proxy_address>:<proxy_port> no-proxy=<cluster cidr>,<service cluster ip range>,127.0.0.1,localhost,<nodes internal ip(s)>/24,<cluster hostname>,.svc,.local
+```
+
+For Example:
+```
+juju config knative-serving http-proxy=http://10.0.13.50:3128/ https-proxy=http://10.0.13.50:3128/ no-proxy=10.1.0.0/16,10.152.183.0/24,127.0.0.1,localhost,10.0.2.0/24,ip-10-0-2-157,.svc,.local
+
+juju config kserve-controller http-proxy=http://10.0.13.50:3128/ https-proxy=http://10.0.13.50:3128/ no-proxy=10.1.0.0/16,10.152.183.0/24,127.0.0.1,localhost,10.0.2.0/24,ip-10-0-2-157,.svc,.local
+```
+
+#### Running using Notebook
+
+##### Prerequistes
+
+Edit the PodDefault `tests/proxy-poddefault.yaml` to replace the placeholders for:
+
+* `http_proxy` and `https_proxy` - The address and port of your proxy server, format should be `<proxy_address>:<proxy_port>`
+* `no_proxy` - A comma separated list of items that should not be proxied. It is recommended to include the following:
+
+`<cluster cidr>,<service cluster ip range>,127.0.0.1,localhost,<nodes internal ip(s)>/24,<cluster hostname>,.svc,.local`
+
+where,
+
+  * `<cluster cidr>`: you can get this value by running:
+
+    ```
+    cat /var/snap/microk8s/current/args/kube-proxy | grep cluster-cidr
+    ```
+
+  * `<service cluster ip range>`: you can get this value by running:
+
+    ```
+    cat /var/snap/microk8s/current/args/kube-apiserver | grep service-cluster-ip-range
+    ```
+
+  * `<nodes internal ip(s)>`: the Internal IP of the nodes where your cluster is running, you can get this value by running:
+
+    ```
+    microk8s kubectl get nodes -o wide
+    ```
+    It is the `INTERNAL-IP` value
+
+  * `<hostname>`: the name of your host on which the cluster is deployed, you can use the `hostname` command to get it
+
+  * `localhost` and `127.0.0.1` are recommended to avoid proxying requests to `localhost`
+
+
+To run the tests behind proxy using Notebook:
+1. Login to the Dashboard and Create a Profile
+2. Apply the PodDefault to your Profile's namespace, make sure you already followed the Prerequisites
+   section to modify the PodDefault. Apply it with:
+   ```
+   kubectl apply -f ./tests/proxy-poddefault.yaml -n <your_namespace>
+   ```
+3. Create a Notebook and from the `Advanced Options > Configurations` select `Add proxy settings`,
+   then click `Launch` to start the Notebook.
+   Wait for the Notebook to be Ready, then Connect to it.
+4. From inside the Notebook, start a new terminal session and clone this repo:
+
+   ```bash
+   git clone https://github.com/canonical/charmed-kubeflow-uats.git
+   ```
+   Open the `charmed-kubeflow-uats/tests` directory and for each `.ipynb` test file there, open it
+   and run the Notebook.
+
+   Currently, the following tests are supported to run behind proxy:
+   * katib
+   * kserve
+   * kfp_v2
+   * training
+
+#### Running using `driver`
+
+You can pass the `--proxy` flag and set the values for proxies to the tox command and this should automatically apply the required changes to run behind proxy.
+
+```bash
+tox -e kubeflow-<local|remote> -- --proxy http_proxy="http_proxy:port" https_proxy="https_proxy:port" no_proxy="<cluster cidr>,<service cluster ip range>,127.0.0.1,localhost,<nodes internal ip(s)>/24,<cluster hostname>,.svc,.local"
+```
+
 #### Developer Notes
 
 Any environment that can be used to access and configure the Charmed Kubeflow deployment is

assets/test-job.yaml.j2

@@ -7,6 +7,9 @@ spec:
   template:
     metadata:
       labels:
+          {% if proxy %}
+          notebook-proxy: "true"
+          {% endif %}
           access-minio: "true"
           access-ml-pipeline: "true"
           mlflow-server-minio: "true"

driver/conftest.py

@@ -10,6 +10,17 @@ def pytest_addoption(parser: Parser):
     * Add a `--filter` option to (de)select test cases based on their name (see also
       https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags)
     """
+    parser.addoption(
+        "--proxy",
+        nargs=3,
+        metavar=("http_proxy", "https_proxy", "no_proxy"),
+        help="Set a number of key-value pairs for the proxy environment variables."
+        " Example: "
+        "--proxy http_proxy='proxy:port' https_proxy='proxy:port' no_proxy=<comma separated of no proxy>'"
+        " If used, a PodDefault will be rendered and applied to the Kubernetes deployment."
+        " It is not used by default.",
+        action="store",
+    )
     parser.addoption(
         "--filter",
         help="Provide a filter to (de)select tests cases based on their name. The filter follows"

driver/test_kubeflow_workloads.py

@@ -5,10 +5,15 @@
 import os
 import subprocess
 from pathlib import Path
+from typing import Dict
 
 import pytest
 from lightkube import ApiError, Client, codecs
-from lightkube.generic_resource import create_global_resource, load_in_cluster_generic_resources
+from lightkube.generic_resource import (
+    create_global_resource,
+    create_namespaced_resource,
+    load_in_cluster_generic_resources,
+)
 from utils import assert_namespace_active, delete_job, fetch_job_logs, wait_for_job
 
 log = logging.getLogger(__name__)
@@ -34,6 +39,14 @@
 
 PYTEST_CMD_BASE = "pytest"
 
+PODDEFAULT_RESOURCE = create_namespaced_resource(
+    group="kubeflow.org",
+    version="v1alpha1",
+    kind="poddefault",
+    plural="poddefaults",
+)
+PODDEFAULT_WITH_PROXY_PATH = Path("tests") / "proxy-poddefault.yaml.j2"
+
 
 @pytest.fixture(scope="session")
 def pytest_filter(request):
@@ -83,6 +96,33 @@ def create_profile(lightkube_client):
     lightkube_client.delete(PROFILE_RESOURCE, name=NAMESPACE)
 
 
+@pytest.fixture(scope="function")
+def create_poddefaults_on_proxy(request, lightkube_client):
+    """Create PodDefault with proxy env variables for the Notebook inside the Job."""
+    # Simply yield if the proxy flag is not set
+    if not request.config.getoption("proxy"):
+        yield
+    else:
+        log.info("Adding PodDefault with proxy settings.")
+        poddefault_resource = codecs.load_all_yaml(
+            PODDEFAULT_WITH_PROXY_PATH.read_text(),
+            context=proxy_context(request),
+        )
+        # Using the first item of the list of poddefault_resource. It is a one item list.
+        lightkube_client.create(poddefault_resource[0], namespace=NAMESPACE)
+
+        yield
+
+        # delete the PodDefault at the end of the module tests
+        log.info("Deleting PodDefault...")
+        poddefault_resource = codecs.load_all_yaml(
+            PODDEFAULT_WITH_PROXY_PATH.read_text(),
+            context=proxy_context(request),
+        )
+        poddefault_name = poddefault_resource[0].metadata.name
+        lightkube_client.delete(PODDEFAULT_RESOURCE, name=poddefault_name, namespace=NAMESPACE)
+
+
 @pytest.mark.abort_on_fail
 async def test_create_profile(lightkube_client, create_profile):
     """Test Profile creation.
@@ -105,7 +145,9 @@ async def test_create_profile(lightkube_client, create_profile):
     assert_namespace_active(lightkube_client, NAMESPACE)
 
 
-def test_kubeflow_workloads(lightkube_client, pytest_cmd, tests_checked_out_commit):
+def test_kubeflow_workloads(
+    lightkube_client, pytest_cmd, tests_checked_out_commit, request, create_poddefaults_on_proxy
+):
     """Run a K8s Job to execute the notebook tests."""
     log.info(f"Starting Kubernetes Job {NAMESPACE}/{JOB_NAME} to run notebook tests...")
     resources = list(
@@ -118,9 +160,11 @@ def test_kubeflow_workloads(lightkube_client, pytest_cmd, tests_checked_out_comm
                 "tests_image": TESTS_IMAGE,
                 "tests_remote_commit": tests_checked_out_commit,
                 "pytest_cmd": pytest_cmd,
+                "proxy": True if request.config.getoption("proxy") else False,
             },
         )
     )
+
     assert len(resources) == 1, f"Expected 1 Job, got {len(resources)}!"
     lightkube_client.create(resources[0], namespace=NAMESPACE)
 
@@ -140,3 +184,12 @@ def teardown_module():
     """Cleanup resources."""
     log.info(f"Deleting Job {NAMESPACE}/{JOB_NAME}...")
     delete_job(JOB_NAME, NAMESPACE)
+
+
+def proxy_context(request) -> Dict[str, str]:
+    """Return a dictionary with proxy environment variables from user input."""
+    proxy_context = {}
+    for proxy in request.config.getoption("proxy"):
+        key, value = proxy.split("=")
+        proxy_context[key] = value
+    return proxy_context