Add a containerized mode to the ECM service #974

Workflow file for this run

.github/workflows/integration-test.yml at 31ee15c

	#
	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#

	name: Integration Test

	on:
	push:
	branches: [master,dev-*]
	pull_request:
	branches: [master,dev-*]

	#concurrency:
	# group: test-${{ github.head_ref \|\| github.run_id }}
	# cancel-in-progress: true

	env:
	KIND_CONFIG_PATH: './linkis-dist/helm/scripts/resources/kind-cluster.yaml'
	KIND_CLUSTER_NAME: 'test-helm'
	MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
	jobs:
	integration-test:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	node-version: [16.0.0]
	kubernetes-version:
	# - 'kindest/node:v1.21.10'
	- 'kindest/node:v1.23.4'
	timeout-minutes: 90
	env:
	TAG: ${{ github.sha }}
	SKIP_TEST: true
	HUB: ghcr.io/apache/linkis
	LINKIS_VERSION: 1.6.0
	steps:
	- name: Free up disk space
	run: \|
	# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
	# du -sh /* 2> /dev/null \| sort -rh 2> /dev/null \| head
	# du -h -d2 /usr 2> /dev/null \| sort -rh 2> /dev/null \| head
	echo $JAVA_HOME
	echo "Check free disk space before cleanup."
	df -h
	echo "Removing non-essential tools and libraries."
	sudo rm -rf "$AGENT_TOOLSDIRECTORY"
	sudo rm -rf /opt/ghc
	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /usr/local/share/boost
	# delete libraries for Android (12G), PowerShell (1.3G), Swift (1.7G)
	sudo rm -rf /usr/local/lib/android
	sudo rm -rf /usr/local/share/powershell
	sudo rm -rf /usr/share/swift
	echo "Check free disk space after cleanup."
	df -h
	echo $JAVA_HOME
	- name: Prune docker images
	run: \|
	echo "Pruning docker images on GH action runner node"
	docker image prune -a -f
	docker system df
	df -h

	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	submodules: true
	- name: Set up JDK 8
	uses: actions/setup-java@v4
	with:
	distribution: 'temurin'
	java-version: 8
	- name: Cache local Maven repository
	uses: actions/cache@v4
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Use Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v4
	with:
	node-version: ${{ matrix.node-version }}
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v1

	- name: Build frontend by node.js
	run: \|
	cd linkis-web
	sed -i "/VUE_APP_MN_CONFIG_PREFIX/d" .env
	npm install
	npm run build

	- name: Build backend by maven
	run: \|
	./mvnw install -Pdocker -Dmysql.connector.scope=compile -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.web=true -Dlinkis.build.ldh=true

	- name: Set up chart-testing
	uses: ./.github/actions/chart-testing-action


	- name: Create Kind cluster
	uses: ./.github/actions/kind-action
	with:
	config: ${{ env.KIND_CONFIG_PATH }}
	node_image: ${{ matrix.kubernetes-version }}
	cluster_name: ${{ env.KIND_CLUSTER_NAME }}

	- name: Start Linkis Service
	run: \|
	docker tag linkis:${{ env.LINKIS_VERSION }} linkis:dev
	docker tag linkis-web:${{ env.LINKIS_VERSION }} linkis-web:dev
	docker tag linkis-ldh:${{ env.LINKIS_VERSION }} linkis-ldh:dev

	# for debug Download the image directly first, then adjust it to active build
	#ROOT_DIR=./linkis-dist/
	#MIRRORS="ghcr.io"
	#TAG="dev"
	#docker pull ${MIRRORS}/apache/linkis/linkis-ldh:${TAG}
	#docker pull ${MIRRORS}/apache/linkis/linkis:${TAG}
	#docker pull ${MIRRORS}/apache/linkis/linkis-web:${TAG}
	#docker tag ${MIRRORS}/apache/linkis/linkis:${TAG} linkis:dev
	#docker tag ${MIRRORS}/apache/linkis/linkis-web:${TAG} linkis-web:dev
	#docker tag ${MIRRORS}/apache/linkis/linkis-ldh:${TAG} linkis-ldh:dev


	#show image list
	docker image ls
	bash ./linkis-dist/helm/scripts/install-mysql.sh false\
	&& bash ./linkis-dist/helm/scripts/install-ldh.sh true \
	&& bash ./linkis-dist/helm/scripts/install-charts-with-ldh.sh linkis linkis-demo true
	n=0
	sleep 60
	while (($n<10))
	do
	kubectl get pods -A
	n=$((n+1))
	sleep 20
	done

	#show linkis pod logs
	#POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-linkismanager -o jsonpath='{.items[0].metadata.name}'`
	#kubectl logs -n linkis ${POD_NAME} -f --tail=10000
	#POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-engineconnmanager -o jsonpath='{.items[0].metadata.name}'`
	#kubectl logs -n linkis ${POD_NAME} -f --tail=10000
	shell: bash

	- name: Linkis-Cli Test
	run: \|
	# Enable port-forward
	bash ./linkis-dist/helm/scripts/remote-proxy.sh start
	# Show port-forward list
	bash ./linkis-dist/helm/scripts/remote-proxy.sh list
	# Check if the web service is available
	curl http://127.0.0.1:8088/indexhtml

	# Execute test by linkis-cli
	POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-mg-gateway -o jsonpath='{.items[0].metadata.name}'`
	kubectl exec -it -n linkis ${POD_NAME} -- bash -c " \
	sh /opt/linkis/bin/linkis-cli --async=true -engineType shell-1 -codeType shell -code \"pwd\" ";

	# Waiting for the task to be executed
	sleep 120

	#kubectl exec -it -n linkis ${POD_NAME} -- bash -c " \
	#sh /opt/linkis/bin/linkis-cli -engineType python-python2 -codeType python -code 'print(\"hello\")' "

	#todo
	#kubectl exec -it -n linkis ${POD_NAME} -- bash -c " \
	#sh /opt/linkis/bin/linkis-cli -engineType hive-3.1.3 -codeType hql -code 'show databases' "

	#kubectl exec -it -n linkis ${POD_NAME} -- bash -c " \
	#sh /opt/linkis/bin/linkis-cli -engineType spark-3.2.1 -codeType sql -code 'show databases' "

	shell: bash

	- name: get Server Log
	run: \|
	# get publicservice log
	POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-ps-publicservice -o jsonpath='{.items[0].metadata.name}'`
	kubectl logs -n linkis ${POD_NAME}

	# get entrance log
	POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-entrance -o jsonpath='{.items[0].metadata.name}'`
	kubectl logs -n linkis ${POD_NAME}

	# get linkismanager log
	POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-linkismanager -o jsonpath='{.items[0].metadata.name}'`
	kubectl logs -n linkis ${POD_NAME}

	# get ecm log
	POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-engineconnmanager -o jsonpath='{.items[0].metadata.name}'`
	kubectl logs -n linkis ${POD_NAME}

	shell: bash

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add a containerized mode to the ECM service #974

Workflow file

Add a containerized mode to the ECM service #974

Jobs

Run details

Workflow file for this run