✨ Add dockerfiles for providers and run the demo using the external p…

…roviders in a Pod (#579) Fixes #470 I have still kept the original base Dockerfile as-is for backwards compatibility. This is the first step in which we are chaging our demo tests to use the external providers. I will create a follow up issue to track the work around actually making the switch in the addon. --------- Signed-off-by: Pranav Gaikwad <[email protected]> Co-authored-by: David Zager <[email protected]>
konveyor · Apr 17, 2024 · 3f6e1bc · 3f6e1bc
1 parent 1e8c3cf
commit 3f6e1bc
Show file tree

Hide file tree

Showing 6 changed files with 184 additions and 21 deletions.
diff --git a/.github/workflows/demo-testing.yml b/.github/workflows/demo-testing.yml
@@ -10,38 +10,97 @@ jobs:
     steps:
       - name: Extract pull request number from inputs or PR description
         id: extract-info
+        env:
+          PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
         run: |
-          echo "${{ github.event.pull_request.body }}"
-          PULL_REQUEST_NUMBER=$(echo "${{ github.event.pull_request.body }}" | grep -oP 'Addon PR: \K\d+' || true)
-          echo "API_TESTS_REF=main" >> $GITHUB_OUTPUT
-          if [ -z "$PULL_REQUEST_NUMBER" ]; then
-            if [ -z "${{ github.event.pull_request.base.ref }}" ]; then
-              echo "Using addon branch main"
-              echo "ADDON_REF=main" >>$GITHUB_ENV
-            else
-              echo "Using addon branch ${{ github.event.pull_request.base.ref }}"
-              echo "ADDON_REF=${{ github.event.pull_request.base.ref }}" >>$GITHUB_ENV
-              # when we know the base branch, we should use the identical branch of api tests
-              echo "API_TESTS_REF=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT
-            fi
+          # if this is a PR, we should use the base branch
+          # else, use the branch on which this is running
+          if [ ! -z ${GITHUB_BASE_REF} ]; then 
+              echo "Using ${GITHUB_BASE_REF}"
+              echo "API_TESTS_REF=${GITHUB_BASE_REF}" >> $GITHUB_OUTPUT
+              echo "ADDON_REF=${GITHUB_BASE_REF}" >>$GITHUB_ENV
           else
-            echo "ADDON_REF=refs/pull/$PULL_REQUEST_NUMBER/merge" >>$GITHUB_ENV
+              echo "Using ${GITHUB_REF_NAME}"
+              echo "API_TESTS_REF=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT
+              echo "ADDON_REF=${GITHUB_REF_NAME}" >>$GITHUB_ENV
           fi
+          
+          # override with explicitely set value in PR description
+          echo "${PULL_REQUEST_BODY}"
+          PULL_REQUEST_NUMBER=$(echo "${PULL_REQUEST_BODY}" | grep -oP '[A|a]ddon [P|p][R|r]: \K\d+' || true)
+          if [ ! -z "$PULL_REQUEST_NUMBER" ]; then
+            echo "Using pull/${PULL_REQUEST_NUMBER} for addon"
+            echo "ADDON_REF=refs/pull/$PULL_REQUEST_NUMBER/merge" >>$GITHUB_ENV
+          fi        
 
       - uses: actions/checkout@v3
 
-      - name: build image
-        run: podman build -t quay.io/konveyor/analyzer-lsp:latest .
+      # build all provider images and the analyzer-lsp image
+      - name: build images
+        run: |
+          function build_image {
+            dir=$1
+            name=$2
+            pushd ${dir}
+            podman build -t quay.io/konveyor/${name}:latest -f Dockerfile .
+            popd
+          }
+          pushd ./external-providers/
+          build_image golang-dependency-provider golang-dependency-provider
+          build_image java-external-provider java-external-provider
+          build_image yq-external-provider yq-external-provider
+          build_image generic-external-provider generic-external-provider
+          popd
+          build_image . analyzer-lsp
+          
+
+      # run the demo in a podman pod
+      - name: run demo image
+        run: |
+          function run_provider {
+            name=$1
+            img=$2
+            port=$3
+            podman run -d --pod analyzer --name ${name} -v test-data:/analyzer-lsp/examples/:Z \
+              quay.io/konveyor/${img}:latest --port ${port}
+          }
+          
+          mkdir /tmp/examples/ && \
+            cp -r examples/* /tmp/examples/ && \
+            cp -r ./external-providers/java-external-provider/examples/* /tmp/examples/
+          podman volume create test-data
+          podman run --rm -v test-data:/target -v /tmp/examples/:/src/:Z \
+            --entrypoint=cp alpine -a /src/. /target/
+          podman pod create --name=analyzer
+          run_provider golang generic-external-provider 9999
+          run_provider nodejs generic-external-provider 9998
+          run_provider python generic-external-provider 9997
+          run_provider java java-external-provider 9996
+          run_provider yq yq-external-provider 9995
+
+          jq 'map(
+            if .name == "go" then del(.binaryPath) | .address = "localhost:9999"
+            elif .name == "nodejs" then del(.binaryPath) | .address = "localhost:9998"
+            elif .name == "python" then del(.binaryPath) | .address = "localhost:9997"
+            elif .name == "java" then del(.binaryPath) | .address = "localhost:9996"
+            elif .name == "yaml" then del(.binaryPath) | .address = "localhost:9995"
+            else .
+            end
+          )' provider_container_settings.json | tee provider_container_settings.json
+
+          podman build -f demo.Dockerfile -t localhost/testing:latest
+          podman run --entrypoint /usr/local/bin/konveyor-analyzer \
+            --pod=analyzer \
+            -v $(pwd)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z \
+            -v $(pwd)/demo-output.yaml:/analyzer-lsp/output.yaml:Z \
+            localhost/testing:latest --dep-output-file=demo-dep-output.yaml
 
-      - name: build demo image
-        run: podman build -f demo.Dockerfile -t localhost/testing:latest
 
       - name: install yq for testing
         run: go install github.com/mikefarah/yq/v4@latest
 
-      - name: run demo image and ensure violations output unchanged
+      - name: ensure violation and dependency outputs are unchanged
         run: |
-          podman run --entrypoint /usr/local/bin/konveyor-analyzer -v $(pwd)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(pwd)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --dep-output-file=demo-dep-output.yaml
           diff \
             <(yq -P 'sort_keys(..)' -o=props <(git show HEAD:demo-output.yaml)) \
             <(yq -P 'sort_keys(..)' -o=props <(cat demo-output.yaml))

diff --git a/.github/workflows/image-build.yaml b/.github/workflows/image-build.yaml
@@ -30,6 +30,22 @@ jobs:
             containerfile: "./external-providers/dotnet-external-provider/Dockerfile"
             context: "./external-providers/dotnet-external-provider"
             architectures: '[ "amd64", "arm64" ]'
+          - name: golang-dependency-provider
+            containerfile: "./external-providers/golang-dependency-provider/Dockerfile"
+            context: "./external-providers/golang-dependency-provider"
+            architectures: '[ "amd64", "arm64" ]'
+          - name: yq-external-provider
+            containerfile: "./external-providers/yq-external-provider/Dockerfile"
+            context: "./external-providers/yq-external-provider"
+            architectures: '[ "amd64", "arm64" ]'
+          - name: java-external-provider
+            containerfile: "./external-providers/java-external-provider/Dockerfile"
+            context: "./external-providers/java-external-provider"
+            architectures: '[ "amd64", "arm64" ]'
+            pre_build_cmd: |
+              TAG=${GITHUB_REF_NAME/main/latest}
+              sed -i "s,FROM quay.io/konveyor/jdtls-server-base,FROM quay.io/konveyor/jdtls-server-base:${TAG}," Dockerfile
+
     uses: konveyor/release-tools/.github/workflows/build-push-images.yaml@main
     with:
       registry: "quay.io/konveyor"
@@ -41,3 +57,19 @@ jobs:
     secrets:
       registry_username: ${{ secrets.QUAY_PUBLISH_ROBOT }}
       registry_password: ${{ secrets.QUAY_PUBLISH_TOKEN }}
+
+  # generic provider requires go dep provider we built in matrix and needs to be sequential
+  generic-external-provider-build:
+    needs: image-build
+    uses: konveyor/release-tools/.github/workflows/build-push-images.yaml@main
+    with:
+      registry: "quay.io/konveyor"
+      image_name: generic-external-provider
+      containerfile: "./external-providers/generic-external-provider/Dockerfile"
+      architectures: '[ "amd64", "arm64" ]'
+      context: "./external-providers/generic-external-provider/"
+      pre_build_cmd: |
+        TAG=${GITHUB_REF_NAME/main/latest}
+        sed -i "s,FROM quay.io/konveyor/golang-dependency-provider,FROM quay.io/konveyor/golang-dependency-provider:${TAG}," Dockerfile
+      
+    
diff --git a/external-providers/generic-external-provider/Dockerfile b/external-providers/generic-external-provider/Dockerfile
@@ -0,0 +1,30 @@
+FROM golang:1.20 as go-builder
+
+WORKDIR /generic-external-provider
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY main.go main.go
+COPY pkg/ pkg/
+RUN go build -o generic-external-provider main.go
+
+FROM quay.io/konveyor/golang-dependency-provider as go-dep-provider
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+
+ENV NODEJS_VERSION=18
+RUN echo -e "[nodejs]\nname=nodejs\nstream=${NODEJS_VERSION}\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module
+RUN microdnf install gcc-c++ python-devel go-toolset python3-devel nodejs -y && \
+    microdnf clean all && \
+    rm -rf /var/cache/dnf
+RUN python3 -m ensurepip --upgrade
+RUN python3 -m pip install 'python-lsp-server>=1.8.2'
+RUN npm install -g typescript-language-server typescript
+RUN go install golang.org/x/tools/gopls@latest
+
+COPY --from=go-builder /generic-external-provider/generic-external-provider /usr/local/bin/generic-external-provider
+COPY --from=go-dep-provider /usr/local/bin/go-dependency-provider /usr/local/bin/go-dependency-provider
+
+ENTRYPOINT [ "/usr/local/bin/generic-external-provider" ]
diff --git a/external-providers/golang-dependency-provider/Dockerfile b/external-providers/golang-dependency-provider/Dockerfile
@@ -0,0 +1,16 @@
+FROM golang:1.20 as go-builder
+
+WORKDIR /go-dependency-provider
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY main.go main.go
+RUN go build -o go-dependency-provider main.go
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+
+COPY --from=go-builder /go-dependency-provider/go-dependency-provider /usr/local/bin/go-dependency-provider
+
+ENTRYPOINT [ "/usr/local/bin/go-dependency-provider" ]
diff --git a/external-providers/java-external-provider/Dockerfile b/external-providers/java-external-provider/Dockerfile
@@ -11,7 +11,7 @@ RUN go mod download
 COPY main.go main.go
 COPY pkg/ pkg/
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o java-external-provider main.go
+RUN go build -a -o java-external-provider main.go
 
 FROM quay.io/konveyor/jdtls-server-base
 

diff --git a/external-providers/yq-external-provider/Dockerfile b/external-providers/yq-external-provider/Dockerfile
@@ -0,0 +1,26 @@
+FROM golang:1.20 as go-builder
+
+WORKDIR /yq-external-provider
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY main.go main.go
+COPY pkg/ pkg/
+
+RUN go build -o yq-external-provider main.go
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+
+RUN microdnf install -y wget tar xz gzip && \
+    microdnf clean all
+ARG TARGETARCH
+ARG YQ_VERSION="v4.40.5"
+ARG YQ_BINARY="yq_linux_${TARGETARCH}"
+RUN wget "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY}.tar.gz" -O - | tar xz && \
+    mv ${YQ_BINARY} /usr/local/bin/yq
+
+COPY --from=go-builder /yq-external-provider/yq-external-provider /usr/local/bin/yq-external-provider
+
+ENTRYPOINT [ "/usr/local/bin/yq-external-provider" ]