Merge pull request #14 from dwolfson/base_compose

Updates to make the widgets easier to use and provide alternate pytho…

README.md

@@ -12,7 +12,10 @@ Please feel free to engage with the community on our slack channel - we'd love y
 For a quick and simple environment to explore some of Egeria's base capabilities, the **egeria-platform.yml**  Docker Compose
 deployment may be a good starting point. Once this script executes successfully, you will have two docker containers running. One for the Egeria platform and one for Kafka. With this running configuration, you can work with any of Egeria's standard interfaces - java APIs, python APIs, or just plain RESTful http calls - and of course, to make use of tools and interfaces that have been built using these APIs.
 
-The set of **Docker Compose** configurations will grow and evolve over time to cover additional scenarios. 
+If you are a data scientist or just want to work within a Jupyter Notebook environment, the `egeria-platform-jupyter-compose` script
+adds a Jupyter server pre-configured with the Egeria Python client, **pyegeria** for you to use.
+The set of **Docker Compose** configurations will continue to grow and evolve over time to cover additional scenarios based on
+community feedback. 
 
 # Contents
 

egeria-platform-jupyter-compose/Dockerfile-jupyter

@@ -7,21 +7,53 @@
 # The published image tag is taken from the numerical version of
 # our base image, and appended with the contents of .tag-append (file)
 FROM quay.io/jupyter/scipy-notebook
+ARG env_name=python312
+ARG py_ver=3.12
 
-RUN pip install --no-cache-dir 'pyegeria>=0.5.5.13'
-RUN pip install --no-cache-dir 'rich'
-RUN pip install --no-cache-dir 'unitycatalog'
-RUN pip install --no-cache-dir 'pipx'
-RUN pipx ensurepath
-RUN pipx install pyegeria
-RUN pipx ensurepath
+# Experimenting moving to latest python
+RUN mamba create --yes -p "${CONDA_DIR}/envs/${env_name}" \
+    python=${py_ver} \
+    'ipykernel' \
+    'jupyterlab' && \
+    mamba clean --all -f -y
 
-RUN alias view_integ='view_integ_daemon_status --integ_url https://host.docker.internal:9443 --view_url https://host.docker.internal:9443'
-RUN alias view_plat='view_platform_status --url https://host.docker.internal:9443'
+#COPY --chown=${NB_UID}:${NB_GID} environment.yml /tmp/
+#RUN mamba env create -p "${CONDA_DIR}/envs/${env_name}" -f /tmp/environment.yml && \
+#    mamba clean --all -f -y
+
+
+RUN "${CONDA_DIR}/envs/${env_name}/bin/python" -m ipykernel install --user --name="${env_name}" && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
+
+RUN "${CONDA_DIR}/envs/${env_name}/bin/pip3" install --no-cache-dir 'pyegeria>=0.5.5.16'
+RUN "${CONDA_DIR}/envs/${env_name}/bin/pip3" install --no-cache-dir 'rich'
+RUN "${CONDA_DIR}/envs/${env_name}/bin/pip3" install --no-cache-dir 'unitycatalog'
+RUN "${CONDA_DIR}/envs/${env_name}/bin/pip3" install --no-cache-dir 'pipx'
+RUN "${CONDA_DIR}/envs/${env_name}/bin/pipx" install pyegeria
+RUN "${CONDA_DIR}/envs/${env_name}/bin/pipx" ensurepath
+
+USER root
+RUN \
+     # This changes a startup hook, which will activate the custom environment for the process
+     echo conda activate "${env_name}" >> /usr/local/bin/before-notebook.d/10activate-conda-env.sh && \
+     # This makes the custom environment default in Jupyter Terminals for all users which might be created later
+     echo conda activate "${env_name}" >> /etc/skel/.bashrc && \
+     # This makes the custom environment default in Jupyter Terminals for already existing NB_USER
+     echo conda activate "${env_name}" >> "/home/${NB_USER}/.bashrc"
+
+USER ${NB_UID}
+#RUN pip install --no-cache-dir 'pyegeria>=0.5.5.15'
+#RUN pip install --no-cache-dir 'rich'
+#RUN pip install --no-cache-dir 'unitycatalog'
+#RUN pip install --no-cache-dir 'pipx'
+#RUN pipx ensurepath
+#RUN pipx install pyegeria
+#RUN pipx ensurepath
 
 RUN mkdir -p /home/jovyan/workspaces
 RUN chmod +xrw /home/jovyan/workspaces
-copy ../Start-Here.md /home/jovyan
+COPY ../Start-Here.md /home/jovyan
 #    && \
 #    fix-permissions "${CONDA_DIR}" && \
 #    fix-permissions "/home/${NB_USER}" \

egeria-platform-jupyter-compose/before-notebook.d/config-coco.sh

@@ -15,4 +15,4 @@
 #python3 /home/jovyan/egeria-labs/common/config_coco_development.py
 #echo "Launching Jupyter notebook server.."
 
-
+/opt/conda/bin/activate python312

egeria-platform-jupyter-compose/egeria-platform-jupyter-compose.yaml

@@ -52,8 +52,6 @@ services:
              - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
              - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
              - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
-
-
 
     egeria-main:
         depends_on:
@@ -77,8 +75,10 @@ services:
             start_interval: 5s
 
         volumes:
-            - ../landing_area:/deployments/landing-area
-            - ../logs:/deployments/logs
+            - ../landing-area:/deployments/landing-area
+            - ../distribution-hub:/deployments/distribution-hub
+            - ../distribution-hub/surveys:/deployments/surveys
+            - ../distribution-hub/logs:/deployments/logs
     #            - "YOUR EXTRA LIBRARY DIRECTORY"/extra:/deployments/extra
     #            - "ANOTHER DIRECTOR TO MOUNT"/:/deployments/user_mount
 

egeria-platform-jupyter-compose/z-egeria-platform-jupyter-compose.yaml

@@ -0,0 +1,95 @@
+---
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Contributors to the Egeria project
+
+
+# To run
+#   * Ensure Docker is installed and running
+#   * Start Egeria Platform stack from the 'coco-lab-docker-compose' folder using:
+#     'docker compose -f ./egeria-platform-jupyter-uc-compose.yaml up --build'
+#   * Subsequent startups can be performed without the '--build' at the end
+#   * Uninstall the containers by issuing 'docker compose -f ./egeria-platform-jupyter-uc-compose.yaml down'
+#
+# Assumptions:
+#   * Ports 7443, 7444, 7445 and 9192 are available on the host system - these are the default ports for Kafka and Egeria.
+#   * by default, the jupyter notebooks in the 'coco-jupyter-labs' folder are mounted and available for use by the jupyter noteboos
+#   * by default, strict SSL validation is turned off
+#
+#
+
+services:
+    jupyter-hub:
+        depends_on:
+            egeria-main:
+                condition: service_healthy
+
+        image: quay.io/jupyter/scipy-notebook
+        container_name: jupyter-work
+        ports:
+            - 8888:8888
+        environment:
+            JUPYTER_ENABLE_LAB: "yes"
+            JUPYTER_TOKEN: "egeria"
+        build:
+            dockerfile: Dockerfile-jupyter
+
+        volumes:
+            - ./before-notebook.d:/usr/local/bin/before-notebook.d
+            - ../work:/home/jovyan/work
+            - ../workspaces:/home/jovyan/workspaces
+
+
+    kafka:
+         image: 'bitnami/kafka:latest'
+         ports:
+             - '9092:9092'
+
+         environment:
+             - KAFKA_CFG_NODE_ID=0
+             - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+             - KAFKA_CFG_PROCESS_ROLES=controller,broker
+             - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+             - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+             - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+             - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+
+
+
+    egeria-main:
+        depends_on:
+            - kafka
+        image: 'egeria-platform:uc1'
+#        image: 'docker.io/odpi/egeria-platform:stable'
+        ports:
+            - '9443:9443'
+        environment:
+            - XTDB_ENABLE_BYTEUTILS_SHA1=True
+            - XTDB_DISABLE_LIBCRYPTO=True
+            - startup.server.list=active-metadata-store,engine-host,integration-daemon,view-server,simple-metadata-store
+            - server.port=9443
+
+        healthcheck:
+            test: curl -k -X GET "https://localhost:9443/open-metadata/platform-services/users/garygeeke/server-platform/origin" || exit 1
+            interval: 20s
+            timeout: 10s
+            retries: 3
+            start_period: 10s
+            start_interval: 5s
+
+        volumes:
+            - ../landing_area:/deployments/landing-area
+            - ../logs:/deployments/logs
+    #            - "YOUR EXTRA LIBRARY DIRECTORY"/extra:/deployments/extra
+    #            - "ANOTHER DIRECTOR TO MOUNT"/:/deployments/user_mount
+
+
+#
+# Change external to true and create volumes manually if you wish to persist between runs
+#
+volumes:
+    zookeeper-data:
+        external: false
+    kafka-data:
+        external: false
+    db_home:
+        external: false