Export Milvus log (#82)

Signed-off-by: zhuwenxing <[email protected]>

.github/workflows/main.yaml

@@ -257,7 +257,19 @@ jobs:
           docker-compose ps -a || true
           kubectl get pods -n default || true
           
-
+      - name: Export logs
+        if: ${{ always() }}
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          if [ ${{ matrix.deploy_tools}} == "helm" ]; then
+            bash ../../scripts/export_log_k8s.sh default milvus-backup logs
+          fi
+          
+          if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
+            bash ../../scripts/export_log_docker.sh logs
+          fi    
+      
       - name: Upload logs
         if: ${{ ! success() }}
         uses: actions/upload-artifact@v2
@@ -267,5 +279,6 @@ jobs:
             ./logs
             ./server.log
             /tmp/ci_logs
+            deployment/${{ matrix.milvus_mode }}/logs
 
 

scripts/export_log_docker.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Exit immediately for non zero status
+set -e
+
+log_dir=${1:-"logs"}
+array=($(docker-compose ps -a|awk 'NR == 1 {next} {print $1}'))
+echo ${array[@]}
+if [ ! -d $log_dir ];
+then
+    mkdir -p $log_dir
+fi
+echo "export logs start"
+for container in ${array[*]}
+do
+if [[ $container == milvus-* ]];
+then
+    echo "export logs for container $container "
+    docker logs $container > ./$log_dir/$container.log 2>&1 || echo "export logs for container $container failed"
+fi
+done
+echo "export logs done"

scripts/export_log_k8s.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Exit immediately for non zero status
+set -e
+
+ns_name=$1
+instance_name=$2
+log_dir=${3:-"k8s_logs"}
+
+#show proxy pod log
+array=($(kubectl get pod -n ${ns_name} -l "component=proxy, app.kubernetes.io/instance=${instance_name}"| awk 'NR == 1 {next} {print $1}'))
+echo ${array[@]}
+
+for pod in ${array[*]}
+do
+echo "show log of proxy pod $pod "
+kubectl logs $pod -n ${ns_name} --tail=100 || echo "show log for pod $pod failed"
+done
+
+# export info of etcd
+array=($(kubectl get pod -n ${ns_name} -l "app.kubernetes.io/name=etcd, app.kubernetes.io/instance=${instance_name}"| awk 'NR == 1 {next} {print $1}'))
+echo ${array[@]}
+mkdir -p $log_dir/etcd_session
+for pod in ${array[*]}
+do
+echo "check session for etcd pod $pod "
+kubectl exec $pod -n ${ns_name} -- etcdctl get --prefix by-dev/meta/session > ./$log_dir/etcd_session/$pod.log || echo "export session for pod $pod failed"
+done
+echo "check session done"
+
+# export logs of all pods
+array_1=($(kubectl get pod -n ${ns_name} -l "app.kubernetes.io/instance=${instance_name}"| awk 'NR == 1 {next} {print $1}'))
+array_2=($(kubectl get pod -n ${ns_name} -l "release=${instance_name}"| awk 'NR == 1 {next} {print $1}'))
+array=(${array_1[@]} ${array_2[@]})
+
+echo ${array[@]}
+if [ ! -d $log_dir/pod_log ] || [ ! -d $log_dir/pod_describe ];
+then
+    mkdir -p $log_dir/pod_log
+    mkdir -p $log_dir/pod_log_previous
+    mkdir -p $log_dir/pod_describe
+fi
+echo "export logs start"
+for pod in ${array[*]}
+do
+echo "export logs for pod $pod "
+kubectl logs $pod -n ${ns_name} > ./$log_dir/pod_log/$pod.log 2>&1
+kubectl logs $pod --previous -n ${ns_name} > ./$log_dir/pod_log_previous/$pod.log 2>&1 || echo "pod $pod has no previous log"
+kubectl describe pod $pod -n ${ns_name} > ./$log_dir/pod_describe/$pod.log
+done
+echo "export logs done"