Merge branch 'main' into feat/generategraphscript

.github/workflows/ci.yml

@@ -2,6 +2,17 @@ name: CI
 
 on:
   workflow_call:
+    inputs:
+      python-versions:
+        description: "Python Versions"
+        required: false
+        type: string
+        default: "['3.10']"
+      frontend-tests-folder:
+        description: "Frontend Tests Folder"
+        required: false
+        type: string
+        default: "tests/core"
   workflow_dispatch:
     inputs:
       branch:
@@ -16,20 +27,24 @@ on:
         description: "Store API Key"
         required: false
         type: string
+      python-versions:
+        description: "Python Versions"
+        required: false
+        type: string
+        default: "['3.10']"
   pull_request:
     types: [synchronize, labeled]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-
 jobs:
   set-ci-condition:
     name: Should Run CI
     runs-on: ubuntu-latest
     outputs:
-      should-run-ci:  ${{ (contains( github.event.pull_request.labels.*.name, 'lgtm') && github.event.pull_request.draft == false) || (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') }}
+      should-run-ci: ${{ (contains( github.event.pull_request.labels.*.name, 'lgtm') && github.event.pull_request.draft == false) || (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') }}
     steps:
       #  Do anything just to make the job run
       - run: echo "Debug CI Condition"
@@ -63,12 +78,16 @@ jobs:
     name: Run Backend Tests
     if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
     uses: ./.github/workflows/python_test.yml
+    with:
+      python-versions: ${{ inputs.python-versions || '["3.10"]' }}
 
   test-frontend:
     needs: path-filter
     name: Run Frontend Tests
     if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.frontend == 'true' || needs.path-filter.outputs.tests == 'true' }}
     uses: ./.github/workflows/typescript_test.yml
+    with:
+      tests_folder: ${{ inputs.frontend-tests-folder }}
     secrets:
       OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
       STORE_API_KEY: "${{ secrets.STORE_API_KEY }}"
@@ -89,7 +108,15 @@ jobs:
   # https://github.com/langchain-ai/langchain/blob/master/.github/workflows/check_diffs.yml
   ci_success:
     name: "CI Success"
-    needs: [test-backend, test-frontend, lint-backend, test-docs-build, set-ci-condition ]
+    needs:
+      [
+        test-backend,
+        test-frontend,
+        lint-backend,
+        test-docs-build,
+        set-ci-condition,
+      ]
+
     if: always()
     runs-on: ubuntu-latest
     env:
@@ -102,4 +129,4 @@ jobs:
           echo $JOBS_JSON
           echo $RESULTS_JSON
           echo "Exiting with $EXIT_CODE"
-          exit $EXIT_CODE
+          exit $EXIT_CODE

.github/workflows/conventional-labels.yml

@@ -15,7 +15,6 @@ jobs:
         uses: Namchee/[email protected]
         with:
           access_token: ${{ secrets.GITHUB_TOKEN }}
-          verbose: true
           issue: false
 
   label:

.github/workflows/docker-build.yml

@@ -3,7 +3,7 @@ on:
   workflow_call:
     inputs:
       version:
-        required: true
+        required: false
         type: string
       release_type:
         required: true
@@ -12,110 +12,245 @@ on:
         required: false
         type: boolean
         default: false
-
+      nightly_tag_main:
+        description: "Tag for the nightly main build"
+        required: false
+        type: string
+        default: ''
+      nightly_tag_base:
+        description: "Tag for the nightly base build"
+        required: false
+        type: string
+        default: ''
+
   workflow_dispatch:
     inputs:
       version:
-        required: true
+        description: "Version to build"
+        required: false
         type: string
       release_type:
+        description: "Type of release"
         required: true
-        type: choice
-        options:
-          - base
-          - main
+        type: string
       pre_release:
+        description: "Pre-release"
         required: false
         type: boolean
-        default: true
+        default: false
+      nightly_tag_main:
+        description: "Tag for the nightly main build"
+        required: false
+        type: string
+        default: ''
+      nightly_tag_base:
+        description: "Tag for the nightly base build"
+        required: false
+        type: string
+        default: ''
+
 env:
   POETRY_VERSION: "1.8.2"
   TEST_TAG: "langflowai/langflow:test"
 
 jobs:
+  get-version:
+    name: Get Version
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get-version-input.outputs.version || steps.get-version-base.outputs.version || steps.get-version-main.outputs.version }}
+      nightly-tag: ${{ steps.resolve-nightly-tag.outputs.nightly_tag }}
+      nightly-build: ${{ steps.resolve-nightly-tag.outputs.nightly_build }}
+    steps:
+      - name: Resolve nightly tag
+        id: resolve-nightly-tag
+        run: |
+          if [[ "${{ inputs.nightly_tag_main }}" != '' ]]; then
+            echo "nightly_tag=${{ inputs.nightly_tag_main }}" >> $GITHUB_OUTPUT
+            echo "nightly_build=true" >> $GITHUB_OUTPUT
+          elif [[ "${{ inputs.nightly_tag_base }}" != '' ]]; then
+            echo "nightly_tag=${{ inputs.nightly_tag_base }}" >> $GITHUB_OUTPUT
+            echo "nightly_build=true" >> $GITHUB_OUTPUT
+          else
+            echo "nightly_tag=" >> $GITHUB_OUTPUT
+            echo "nightly_build=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Check out the code at a specific ref
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.nightly_tag_main || github.ref }}
+      - name: Set up Python 3.12 + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_caching"
+        with:
+          python-version: "3.12"
+          poetry-version: ${{ env.POETRY_VERSION }}
+          cache-key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
+      - name: Get Version from Input
+        if: ${{ inputs.version != '' }}
+        id: get-version-input
+        run: |
+          version=${{ inputs.version }}
+          echo version=$version >> $GITHUB_OUTPUT
+      - name: Get Version Base
+        if: ${{ inputs.version == '' && inputs.release_type == 'base' }}
+        id: get-version-base
+        run: |
+          version=$(cd src/backend/base && poetry version --short)
+          echo version=$version >> $GITHUB_OUTPUT
+      - name: Get Version Main
+        if: ${{ inputs.version == '' && inputs.release_type == 'main' }}
+        id: get-version-main
+        run: |
+          version=$(poetry version --short)
+          echo version=$version >> $GITHUB_OUTPUT
   setup:
     runs-on: ubuntu-latest
+    needs: get-version
     outputs:
-      tags: ${{ steps.set-vars.outputs.tags }}
+      docker_tags: ${{ steps.set-vars.outputs.docker_tags }}
+      ghcr_tags: ${{ steps.set-vars.outputs.ghcr_tags }}
       file: ${{ steps.set-vars.outputs.file }}
+    env:
+      NIGHTLY_TAG: ${{ needs.get-version.outputs.nightly-tag }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out the code at a specific ref
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.nightly_tag_main || github.ref }}
+
       - name: Set Dockerfile and Tags
         id: set-vars
         run: |
+          nightly_suffix=''
+          if [[ "${{ env.NIGHTLY_TAG }}" != '' ]]; then
+            nightly_suffix="-nightly"
+          fi
+
           if [[ "${{ inputs.release_type }}" == "base" ]]; then
-            echo "tags=langflowai/langflow:base-${{ inputs.version }},langflowai/langflow:base-latest" >> $GITHUB_OUTPUT
+            echo "docker_tags=langflowai/langflow${nightly_suffix}:base-${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:base-latest" >> $GITHUB_OUTPUT
+            echo "ghcr_tags=ghcr.io/langflow-ai/langflow${nightly_suffix}:base-${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow${nightly_suffix}:base-latest" >> $GITHUB_OUTPUT
             echo "file=./docker/build_and_push_base.Dockerfile" >> $GITHUB_OUTPUT
           else
             if [[ "${{ inputs.pre_release }}" == "true" ]]; then
-              echo "tags=langflowai/langflow:${{ inputs.version }}" >> $GITHUB_OUTPUT
+              echo "docker_tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }}" >> $GITHUB_OUTPUT
+              echo "ghcr_tags=ghcr.io/langflow-ai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }}" >> $GITHUB_OUTPUT
             else
-              echo "tags=langflowai/langflow:${{ inputs.version }},langflowai/langflow:latest" >> $GITHUB_OUTPUT
+              echo "docker_tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT
+              echo "ghcr_tags=ghcr.io/langflow-ai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT
             fi
             echo "file=./docker/build_and_push.Dockerfile" >> $GITHUB_OUTPUT
           fi
   build:
     runs-on: ubuntu-latest
-    needs: setup
+    needs: [get-version, setup]
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out the code at a specific ref
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.nightly_tag_main || github.ref }} 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and Push Docker Image
+      - name: Build and Push to Docker Hub
         uses: docker/build-push-action@v6
         with:
           context: .
           push: true
           file: ${{ needs.setup.outputs.file }}
-          tags: ${{ needs.setup.outputs.tags }}
+          tags: ${{ needs.setup.outputs.docker_tags }}
+          # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
+          provenance: false
+
+      - name: Login to Github Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.TEMP_GHCR_TOKEN}}
+      - name: Build and push to Github Container Registry
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          file: ${{ needs.setup.outputs.file }}
+          tags: ${{ needs.setup.outputs.ghcr_tags }}
           # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
           provenance: false
 
   build_components:
     if: ${{ inputs.release_type == 'main' }}
     runs-on: ubuntu-latest
-    needs: build
+    permissions:
+      packages: write
+    needs: [build, get-version]
     strategy:
       matrix:
-        component: [backend, frontend]
+        component: [docker-backend, docker-frontend, ghcr-backend, ghcr-frontend]
         include:
-          - component: backend
+          - component: docker-backend
+            dockerfile: ./docker/build_and_push_backend.Dockerfile
+            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend{0}:{1}', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) || format('langflowai/langflow-backend{0}:{1},langflowai/langflow-backend{0}:latest',  needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) }}
+            langflow_image: langflowai/langflow${{ needs.get-version.outputs.nightly-build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }}
+          - component: docker-frontend
+            dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile
+            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend{0}:{1}', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) || format('langflowai/langflow-frontend{0}:{1},langflowai/langflow-frontend{0}:latest',  needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) }}
+            langflow_image: langflowai/langflow${{ needs.get-version.outputs.nightly-build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }}
+          - component: ghcr-backend
             dockerfile: ./docker/build_and_push_backend.Dockerfile
-            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend:{0}', inputs.version) || format('langflowai/langflow-backend:{0},langflowai/langflow-backend:latest', inputs.version) }}
-          - component: frontend
+            tags: ${{ format('ghcr.io/langflow-ai/langflow-backend{0}:{1},ghcr.io/langflow-ai/langflow-backend{0}:latest', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) }}
+            langflow_image: ghcr.io/langflow-ai/langflow${{ needs.get-version.outputs.nightly-build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }}
+          - component: ghcr-frontend
             dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile
-            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend:{0}', inputs.version) || format('langflowai/langflow-frontend:{0},langflowai/langflow-frontend:latest', inputs.version) }}
+            tags: ${{ format('ghcr.io/langflow-ai/langflow-frontend{0}:{1},ghcr.io/langflow-ai/langflow-frontend{0}:latest', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) }}
+            langflow_image: ghcr.io/langflow-ai/langflow${{ needs.get-version.outputs.nightly-build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out the code at a specific ref
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.nightly_tag_main || github.ref }} 
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Login to Docker Hub
+        if: ${{ matrix.component == 'docker-backend' }} || ${{ matrix.component == 'docker-frontend' }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Wait for Docker Hub to propagate (for backend)
+
+      - name: Login to Github Container Registry
+        if: ${{ matrix.component == 'ghcr-backend' }} || ${{ matrix.component == 'ghcr-frontend' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.TEMP_GHCR_TOKEN}}
+
+      - name: Wait for propagation (for backend)
         run: sleep 120
+
       - name: Build and push ${{ matrix.component }}
         uses: docker/build-push-action@v6
         with:
           context: .
           push: true
           build-args: |
-            LANGFLOW_IMAGE=langflowai/langflow:${{ inputs.version }}
+            LANGFLOW_IMAGE=${{ matrix.langflow_image }}
           file: ${{ matrix.dockerfile }}
           tags: ${{ matrix.tags }}
           # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
           provenance: false
 
   restart-space:
     name: Restart HuggingFace Spaces
-    if: ${{ inputs.release_type == 'main' }}
+    if: ${{ inputs.release_type == 'main' && inputs.nightly_build == 'false' }}
     runs-on: ubuntu-latest
     needs: build
     strategy:

.github/workflows/fetch_docs_notion.yml

@@ -50,7 +50,7 @@ jobs:
 
       - name: Create Pull Request
         id: create_pr
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: Update docs from Notion