Refactored Computing API Components

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# 127 is width of the Github code viewer,
+# black default is 88 so this will only warn about comments >127
+max-line-length = 127
+# Ignore errors due to incompatibility with black
+#https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html
+extend-ignore = E203,E701

.github/workflows/format-lint.yml

@@ -0,0 +1,31 @@
+name: format-and-lint
+
+on: pull_request
+
+jobs:
+  test-and-lint:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.9.16
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.9.16'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Run isort
+        uses: isort/isort-action@master
+      - name: Test formatting with black
+        run: |
+          black . --check

.github/workflows/publish-image.yml

@@ -0,0 +1,45 @@
+name: Create and publish image
+
+on:
+  push:
+    branches: ['main']
+    tags: ['v*']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -1,8 +1,12 @@
+# ggshield
+.cache_ggshield
+
 # Dirctory
 .DS_Store
 content-api/src/__pycache__/*
 content-regist/src/__pycache__/*
 content-regist/src/kwarg_editor_copy.py
+database/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+default_language_version:
+    python: python3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-ast
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: debug-statements
+  - repo: https://github.com/gitguardian/ggshield
+    rev: v1.25.0
+    hooks:
+      - id: ggshield
+        language_version: python3
+        stages: [commit]
+  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.2.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]

LICENSE.txt

@@ -1,4 +1,4 @@
-MLExchange Copyright (c) 2021, The Regents of the University of California,
+MLExchange Copyright (c) 2024, The Regents of the University of California,
 through Lawrence Berkeley National Laboratory (subject to receipt of
 any required approvals from the U.S. Dept. of Energy). All rights reserved.
 

README.md

@@ -1,80 +1,77 @@
 # MLExchange Content Registry
-This is the first Dash 2 version of Content Registry.  
+This is the first Dash 2 version of Content Registry.
 
 Content registry and its API for the MLExchange platform.
 The architecture looks like below.
 
 ![image info](./assets/content_registry_v2.png)
 
-Contents include:  
-	- models  
-	- apps  
-	- workflows  
-	- assets (metadata, e.g., tags, trained models etc.) 
-	- 
+Contents include:
+	- models
+	- apps
+	- workflows
+	- assets (metadata, e.g., tags, trained models etc.)
+	-
 
 ### GUI
 ![image info](./assets/gui.png)
 
 
 ## How to use
 
-Prerequisite: install [mlex\_compute\_api](https://github.com/mlexchange/mlex_computing_api)
+Note: the current version of content registry no longer requires installing [mlex\_compute\_api](https://github.com/mlexchange/mlex_computing_api) as a pre-requisition.
 
 **Note:** This version connects to a local mongodb container named `mongodb` with a port number `27017`. Please checkout the `atlas` branch for using our cloud mongodb service.
 
-**Running content registry**.   
+**Running content registry**.
 
-1. Create the same environmental file (.env) as the one used in the compute api.
+1. Create an environmental file (.env) from the `.env.example` provided in the repository.
 
-	```	
-	MONGO_DB_USERNAME=your_username     
-	MONGO_DB_PASSWORD=your_password               
-	```	   
-2. If it was the first time running MLExchange content registry, you need to initialize its database using the command `./init_db.sh`. If the error msg says 'duplicate key error collection', it means the content database is already existent. 
+	```
+	MONGO_DB_USERNAME=your_username
+	MONGO_DB_PASSWORD=your_password
+	```
 
-	**Note:** to re-initialize the database, it is suggested to first clean `/dump` folder in the local mongodb container.
-
-3. Run `docker-compose up --build`.
+2. Run `docker-compose up --build`.
 
-**Viewing existing contents**  
+**Viewing existing contents**
 Scroll down to the bottom of the page, click on **Refresh** button.
 
-**Delete existing contents**  
+**Delete existing contents**
 Select contents from the table and click **Delete** button.
 
-**Modify existing contents**  
-Currently, the **Modify** button is deprecated. 
+**Modify existing contents**
+Currently, the **Modify** button is deprecated.
 Therefore, the easiest way to modify an existing registration is to delete the content, then either register a new content by filling out the forms or upload the revised JSON document again (drag and drop, click **Validate** button first, then click the **Upload** button).
 
 
 ## Adaptive Dash GUI component
-**name**: a unique string, equivalent to the `id` in each Dash component   
-**param_key**: a parameter key generally used to construct the `cmd` in job dictionary for MLExchange compute api  
+**name**: a unique string, equivalent to the `id` in each Dash component
+**param_key**: a parameter key generally used to construct the `cmd` in job dictionary for MLExchange compute api
 
 ## API calls
-get\_models()  
-get\_model(uid) 
+get\_models()
+get\_model(uid)
 
 
-get\_apps()  
-get\_app(uid) 
+get\_apps()
+get\_app(uid)
 
 
-get\_workflows()  
-get\_workflow(uid) 
+get\_workflows()
+get\_workflow(uid)
 
 
-post\_assets(data)  
-post\_asset(data) 
-get\_assets()  
-get\_asset(uid) 
-delete\_assets(query)  
-delete\_asset(uid) 
+post\_assets(data)
+post\_asset(data)
+get\_assets()
+get\_asset(uid)
+delete\_assets(query)
+delete\_asset(uid)
 
 
 ## License
-MLExchange Copyright (c) 2021, The Regents of the University of California,
+MLExchange Copyright (c) 2024, The Regents of the University of California,
 through Lawrence Berkeley National Laboratory (subject to receipt of
 any required approvals from the U.S. Dept. of Energy). All rights reserved.
 

content-api/docker/Dockerfile

@@ -4,13 +4,13 @@ MAINTAINER THE MLEXCHANGE TEAM
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     python3-pip\
-    libxext6 
+    libxext6
 
 
 COPY docker/requirements.txt requirements.txt
 
 RUN pip3 install --upgrade pip &&\
-    pip3 install --timeout=5000 -r requirements.txt 
+    pip3 install --timeout=5000 -r requirements.txt
 
 
 WORKDIR /app/api
@@ -20,4 +20,4 @@ ENV PYTHONUNBUFFERED=1
 
 CMD bash
 WORKDIR /app/api/src
-CMD uvicorn content_api:app --host 0.0.0.0 --port 8000 --reload
+CMD uvicorn content_api:app --host 0.0.0.0 --port 8000 --reload

content-api/src/api_util.py

@@ -1,10 +1,12 @@
-import os
 import configparser
+import os
+
 import requests
 
 config = configparser.ConfigParser()
 config.read(os.path.join(os.path.dirname(__file__), "config.ini"))
-WEBHOOK_RECEIVER_URL = 'http://%s' % config['webhook']['RECEIVER']
+WEBHOOK_RECEIVER_URL = "http://%s" % config["webhook"]["RECEIVER"]
+
 
 def send_webhook(msg):
     """
@@ -14,23 +16,17 @@ def send_webhook(msg):
     """
     try:
         # Post a webhook message
-        # default is a function applied to objects that are not serializable = it converts them to str
-        resp = requests.post(WEBHOOK_RECEIVER_URL, json=msg, headers={'Content-Type': 'application/json'}, timeout=1.0)
-        # Returns an HTTPError if an error has occurred during the process (used for debugging).
+        resp = requests.post(
+            WEBHOOK_RECEIVER_URL,
+            json=msg,
+            headers={"Content-Type": "application/json"},
+            timeout=1.0,
+        )
+        # Returns an HTTPError if an error has occurred
+        # during the process (used for debugging).
         resp.raise_for_status()
-    except requests.exceptions.HTTPError as err:
-        #print("An HTTP Error occurred",repr(err))
-        pass
-    except requests.exceptions.ConnectionError as err:
-        #print("An Error Connecting to the API occurred", repr(err))
-        pass
-    except requests.exceptions.Timeout as err:
-        #print("A Timeout Error occurred", repr(err))
-        pass
-    except requests.exceptions.RequestException as err:
-        #print("An Unknown Error occurred", repr(err))
-        pass
-    except:
-        pass
+    except Exception as err:
+        print(err)
+        return False
     else:
-        return resp.status_code
+        return resp.ok

content-api/src/config.ini

@@ -8,4 +8,4 @@ RECEIVER = content-api:8000/api/v0/receiver
 USER_API_PORT = 5003
 
 [search api port]
-SEARCH_API_PORT = 8060
+SEARCH_API_PORT = 8060