From e2e8ae5d8cc72c77e036c77e98101ec4a1b07ca4 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 17:23:46 +0800 Subject: [PATCH 01/29] Add hyperlinks and paths validation. Signed-off-by: ZePan110 --- .../workflows/pr-dockerfile-path-scan.yaml | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 125a3e80b..561ff5c21 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -156,3 +156,94 @@ jobs: echo "Please modify the corresponding README in GenAIExamples repo and ask suyue.chen@intel.com for final confirmation." exit 1 fi + + check-the-validity-of-hyperlinks-in-README: + runs-on: ubuntu-latest + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout Repo GenAIExamples + uses: actions/checkout@v4 + + - name: Check the Validity of Hyperlinks + run: | + cd ${{github.workspace}} + fail="FALSE" + url_lines=$(grep -Eo '\]\(http[s]?://[^)]+\)' --include='*.md' -r .) + if [ -n "$url_lines" ]; then + for url_line in $url_lines; do + url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//') + path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) + response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from ${{github.workspace}}/$path: $url" + fail="TRUE" + fi + fi + done + fi + + if [[ "$fail" == "TRUE" ]]; then + exit 1 + else + echo "All hyperlinks are valid." + fi + shell: bash + + check-the-validity-of-relative-path: + runs-on: ubuntu-latest + steps: + - name: Clean up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout Repo GenAIExamples + uses: actions/checkout@v4 + + - name: Checking Relative Path Validity + run: | + cd ${{github.workspace}} + fail="FALSE" + link_head="https://github.com/opea-project/GenAIExamples/blob/main/" + png_lines=$(grep -Eo '\]\([^)]+\)' -r -I .|grep -Ev 'http') + if [ -n "$png_lines" ]; then + for png_line in $png_lines; do + refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-) + png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1) + if [[ "${png_path:0:1}" == "/" ]]; then + check_path=${{github.workspace}}$png_path + else + check_path=${{github.workspace}}/$(dirname "$refer_path")/$png_path + fi + real_path=$(realpath $check_path) + if [ $? -ne 0 ]; then + echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" + fail="TRUE" + else + url=$link_head$(echo "$real_path" | sed 's|.*/GenAIExamples/||') + response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from $check_path: $url" + fail="TRUE" + fi + fi + fi + done + fi + + if [[ "$fail" == "TRUE" ]]; then + exit 1 + else + echo "All hyperlinks are valid." + fi + shell: bash From baac2d4cba337461eff3196abd50f3aa36bd9ef6 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 17:26:13 +0800 Subject: [PATCH 02/29] Fix format issue. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 561ff5c21..ef7782c4c 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -157,7 +157,7 @@ jobs: exit 1 fi - check-the-validity-of-hyperlinks-in-README: + check-the-validity-of-hyperlinks-in-README: runs-on: ubuntu-latest steps: - name: Clean Up Working Directory From dbe26a9d2d6e73a1e9e4603b0fe8ea92af91d061 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 17:40:28 +0800 Subject: [PATCH 03/29] Change runs-on Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index ef7782c4c..d7a0b7539 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -158,7 +158,7 @@ jobs: fi check-the-validity-of-hyperlinks-in-README: - runs-on: ubuntu-latest + runs-on: self-hosted #ubuntu-latest steps: - name: Clean Up Working Directory run: sudo rm -rf ${{github.workspace}}/* @@ -197,7 +197,7 @@ jobs: shell: bash check-the-validity-of-relative-path: - runs-on: ubuntu-latest + runs-on: self-hosted #ubuntu-latest steps: - name: Clean up Working Directory run: sudo rm -rf ${{github.workspace}}/* From a9652e2fc41571a33e8a790b2f8fcfa0b1bea722 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 17:23:46 +0800 Subject: [PATCH 04/29] Add hyperlinks and paths validation. Signed-off-by: ZePan110 --- .../workflows/pr-dockerfile-path-scan.yaml | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 125a3e80b..561ff5c21 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -156,3 +156,94 @@ jobs: echo "Please modify the corresponding README in GenAIExamples repo and ask suyue.chen@intel.com for final confirmation." exit 1 fi + + check-the-validity-of-hyperlinks-in-README: + runs-on: ubuntu-latest + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout Repo GenAIExamples + uses: actions/checkout@v4 + + - name: Check the Validity of Hyperlinks + run: | + cd ${{github.workspace}} + fail="FALSE" + url_lines=$(grep -Eo '\]\(http[s]?://[^)]+\)' --include='*.md' -r .) + if [ -n "$url_lines" ]; then + for url_line in $url_lines; do + url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//') + path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) + response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from ${{github.workspace}}/$path: $url" + fail="TRUE" + fi + fi + done + fi + + if [[ "$fail" == "TRUE" ]]; then + exit 1 + else + echo "All hyperlinks are valid." + fi + shell: bash + + check-the-validity-of-relative-path: + runs-on: ubuntu-latest + steps: + - name: Clean up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout Repo GenAIExamples + uses: actions/checkout@v4 + + - name: Checking Relative Path Validity + run: | + cd ${{github.workspace}} + fail="FALSE" + link_head="https://github.com/opea-project/GenAIExamples/blob/main/" + png_lines=$(grep -Eo '\]\([^)]+\)' -r -I .|grep -Ev 'http') + if [ -n "$png_lines" ]; then + for png_line in $png_lines; do + refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-) + png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1) + if [[ "${png_path:0:1}" == "/" ]]; then + check_path=${{github.workspace}}$png_path + else + check_path=${{github.workspace}}/$(dirname "$refer_path")/$png_path + fi + real_path=$(realpath $check_path) + if [ $? -ne 0 ]; then + echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" + fail="TRUE" + else + url=$link_head$(echo "$real_path" | sed 's|.*/GenAIExamples/||') + response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from $check_path: $url" + fail="TRUE" + fi + fi + fi + done + fi + + if [[ "$fail" == "TRUE" ]]; then + exit 1 + else + echo "All hyperlinks are valid." + fi + shell: bash From 3f17566a8904005e15e31a9c74bded432ee8fb9b Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 17:26:13 +0800 Subject: [PATCH 05/29] Fix format issue. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 561ff5c21..ef7782c4c 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -157,7 +157,7 @@ jobs: exit 1 fi - check-the-validity-of-hyperlinks-in-README: + check-the-validity-of-hyperlinks-in-README: runs-on: ubuntu-latest steps: - name: Clean Up Working Directory From 2824ad9b6db561855a1957a14e3ef1dac655f2df Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 17:40:28 +0800 Subject: [PATCH 06/29] Change runs-on Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index ef7782c4c..d7a0b7539 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -158,7 +158,7 @@ jobs: fi check-the-validity-of-hyperlinks-in-README: - runs-on: ubuntu-latest + runs-on: self-hosted #ubuntu-latest steps: - name: Clean Up Working Directory run: sudo rm -rf ${{github.workspace}}/* @@ -197,7 +197,7 @@ jobs: shell: bash check-the-validity-of-relative-path: - runs-on: ubuntu-latest + runs-on: self-hosted #ubuntu-latest steps: - name: Clean up Working Directory run: sudo rm -rf ${{github.workspace}}/* From c43daee754fa5895707bb8586d4ae860af1000d1 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 18:52:00 +0800 Subject: [PATCH 07/29] Change link head. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index d7a0b7539..736537c9f 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -163,7 +163,7 @@ jobs: - name: Clean Up Working Directory run: sudo rm -rf ${{github.workspace}}/* - - name: Checkout Repo GenAIExamples + - name: Checkout Repo GenAIComps uses: actions/checkout@v4 - name: Check the Validity of Hyperlinks @@ -202,14 +202,14 @@ jobs: - name: Clean up Working Directory run: sudo rm -rf ${{github.workspace}}/* - - name: Checkout Repo GenAIExamples + - name: Checkout Repo GenAIComps uses: actions/checkout@v4 - name: Checking Relative Path Validity run: | cd ${{github.workspace}} fail="FALSE" - link_head="https://github.com/opea-project/GenAIExamples/blob/main/" + link_head="https://github.com/opea-project/GenAIComps/blob/main/" png_lines=$(grep -Eo '\]\([^)]+\)' -r -I .|grep -Ev 'http') if [ -n "$png_lines" ]; then for png_line in $png_lines; do From 259d73caf14dd86de15291f96c1bcf3298cac41c Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 19:11:09 +0800 Subject: [PATCH 08/29] Fix issue. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 736537c9f..a9980cc08 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -225,7 +225,7 @@ jobs: echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" fail="TRUE" else - url=$link_head$(echo "$real_path" | sed 's|.*/GenAIExamples/||') + url=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps/||') response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") if [ "$response" -ne 200 ]; then echo "**********Validation failed, try again**********" From 4aa58dd229c4b5ce4375b1be697fb09ca4495bf7 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 19:18:48 +0800 Subject: [PATCH 09/29] Add output. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index a9980cc08..9aff5da36 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -233,7 +233,7 @@ jobs: if [ "$response_retry" -eq 200 ]; then echo "*****Retry successfully*****" else - echo "Invalid link from $check_path: $url" + echo "Invalid link from $check_path: $url, $png_line" fail="TRUE" fi fi From 456a36b3df29f4cdb7aadfa02f1a4e3de65aec25 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 19:24:43 +0800 Subject: [PATCH 10/29] Change serch rules. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 9aff5da36..9dbac6688 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -210,7 +210,7 @@ jobs: cd ${{github.workspace}} fail="FALSE" link_head="https://github.com/opea-project/GenAIComps/blob/main/" - png_lines=$(grep -Eo '\]\([^)]+\)' -r -I .|grep -Ev 'http') + png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http') if [ -n "$png_lines" ]; then for png_line in $png_lines; do refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-) From 6dca8961dcf429f11e618bd50782ad222bbae811 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 20:56:10 +0800 Subject: [PATCH 11/29] Change output and fix error Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 +- comps/guardrails/llama_guard/langchain/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 9dbac6688..d89f216b1 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -233,7 +233,7 @@ jobs: if [ "$response_retry" -eq 200 ]; then echo "*****Retry successfully*****" else - echo "Invalid link from $check_path: $url, $png_line" + echo "Invalid link from $real_path: $url, $png_line" fail="TRUE" fi fi diff --git a/comps/guardrails/llama_guard/langchain/README.md b/comps/guardrails/llama_guard/langchain/README.md index 869f30850..727fb9c24 100644 --- a/comps/guardrails/llama_guard/langchain/README.md +++ b/comps/guardrails/llama_guard/langchain/README.md @@ -51,7 +51,7 @@ curl 127.0.0.1:8088/generate \ ### 1.4 Start Guardrails Service -Optional: If you have deployed a Guardrails model with TGI Gaudi Service other than default model (i.e., `meta-llama/Meta-Llama-Guard-2-8B`) [from section 1.2](## 1.2 Start TGI Gaudi Service), you will need to add the eviornment variable `SAFETY_GUARD_MODEL_ID` containing the model id. For example, the following informs the Guardrails Service the deployed model used LlamaGuard2: +Optional: If you have deployed a Guardrails model with TGI Gaudi Service other than default model (i.e., `meta-llama/Meta-Llama-Guard-2-8B`) [from section 1.2](#12-start-tgi-gaudi-service), you will need to add the eviornment variable `SAFETY_GUARD_MODEL_ID` containing the model id. For example, the following informs the Guardrails Service the deployed model used LlamaGuard2: ```bash export SAFETY_GUARD_MODEL_ID="meta-llama/Meta-Llama-Guard-2-8B" From 508bc6f4709b61a76b4dffde283345b428a0910a Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 21:07:26 +0800 Subject: [PATCH 12/29] For test Signed-off-by: ZePan110 --- LEGAL_INFORMATION.md | 1 + test | 1 + 2 files changed, 2 insertions(+) create mode 100644 test diff --git a/LEGAL_INFORMATION.md b/LEGAL_INFORMATION.md index 7c71af2b9..d196eb3a0 100644 --- a/LEGAL_INFORMATION.md +++ b/LEGAL_INFORMATION.md @@ -2,6 +2,7 @@ 1. [License](#license) 2. [Citation](#citation) +3. [test](#test) ## License diff --git a/test b/test new file mode 100644 index 000000000..434d4406f --- /dev/null +++ b/test @@ -0,0 +1 @@ +ttttttttttttttttttttttt \ No newline at end of file From 9ca788107efbb08ba1739e7b19742d0245397add Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 21:19:52 +0800 Subject: [PATCH 13/29] Fix error Signed-off-by: ZePan110 --- comps/vectorstores/pathway/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comps/vectorstores/pathway/README.md b/comps/vectorstores/pathway/README.md index 9b46fcd51..dee8fc252 100644 --- a/comps/vectorstores/pathway/README.md +++ b/comps/vectorstores/pathway/README.md @@ -3,7 +3,7 @@ Set the environment variables for Pathway, and the embedding model. > Note: If you are using `TEI_EMBEDDING_ENDPOINT`, make sure embedding service is already running. -> See the instructions under [here](../../../retrievers/langchain/pathway/README.md) +> See the instructions under [here](../../../retrievers/pathway/langchain/README.md) ```bash export PATHWAY_HOST=0.0.0.0 From 0fcaefc8290ee0aac247cb0d902093a22e2effbb Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 21:24:03 +0800 Subject: [PATCH 14/29] Fix error. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 ++ comps/vectorstores/pathway/README.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index d89f216b1..b91734597 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -209,6 +209,7 @@ jobs: run: | cd ${{github.workspace}} fail="FALSE" + branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}" link_head="https://github.com/opea-project/GenAIComps/blob/main/" png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http') if [ -n "$png_lines" ]; then @@ -228,6 +229,7 @@ jobs: url=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps/||') response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") if [ "$response" -ne 200 ]; then + echo "$url" echo "**********Validation failed, try again**********" response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") if [ "$response_retry" -eq 200 ]; then diff --git a/comps/vectorstores/pathway/README.md b/comps/vectorstores/pathway/README.md index dee8fc252..4a30287f0 100644 --- a/comps/vectorstores/pathway/README.md +++ b/comps/vectorstores/pathway/README.md @@ -3,7 +3,7 @@ Set the environment variables for Pathway, and the embedding model. > Note: If you are using `TEI_EMBEDDING_ENDPOINT`, make sure embedding service is already running. -> See the instructions under [here](../../../retrievers/pathway/langchain/README.md) +> See the instructions under [here](../../retrievers/pathway/langchain/README.md) ```bash export PATHWAY_HOST=0.0.0.0 From aea76509ba259a48b40b7455e92d3a2eb36d69fc Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 21:43:12 +0800 Subject: [PATCH 15/29] Fix error. Signed-off-by: ZePan110 --- comps/dataprep/README.md | 2 +- comps/finetuning/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/comps/dataprep/README.md b/comps/dataprep/README.md index 0deb96c3b..46a57d37d 100644 --- a/comps/dataprep/README.md +++ b/comps/dataprep/README.md @@ -11,7 +11,7 @@ apt-get install libreoffice ## Use LVM (Large Vision Model) for Summarizing Image Data -Occasionally unstructured data will contain image data, to convert the image data to the text data, LVM can be used to summarize the image. To leverage LVM, please refer to this [readme](../lvms/README.md) to start the LVM microservice first and then set the below environment variable, before starting any dataprep microservice. +Occasionally unstructured data will contain image data, to convert the image data to the text data, LVM can be used to summarize the image. To leverage LVM, please refer to this [readme](../lvms/llava/README.md) to start the LVM microservice first and then set the below environment variable, before starting any dataprep microservice. ```bash export SUMMARIZE_IMAGE_VIA_LVM=1 diff --git a/comps/finetuning/README.md b/comps/finetuning/README.md index 21fd585d6..93b65a3e8 100644 --- a/comps/finetuning/README.md +++ b/comps/finetuning/README.md @@ -219,7 +219,7 @@ curl http://${your_ip}:8015/v1/finetune/list_checkpoints -X POST -H "Content-Typ ### 3.4 Leverage fine-tuned model -After fine-tuning job is done, fine-tuned model can be chosen from listed checkpoints, then the fine-tuned model can be used in other microservices. For example, fine-tuned reranking model can be used in [reranks](../reranks/README.md) microservice by assign its path to the environment variable `RERANK_MODEL_ID`, fine-tuned embedding model can be used in [embeddings](../embeddings/README.md) microservice by assign its path to the environment variable `model`, LLMs after instruction tuning can be used in [llms](../llms/README.md) microservice by assign its path to the environment variable `your_hf_llm_model`. +After fine-tuning job is done, fine-tuned model can be chosen from listed checkpoints, then the fine-tuned model can be used in other microservices. For example, fine-tuned reranking model can be used in [reranks](../reranks/fastrag/README.md) microservice by assign its path to the environment variable `RERANK_MODEL_ID`, fine-tuned embedding model can be used in [embeddings](../embeddings/README.md) microservice by assign its path to the environment variable `model`, LLMs after instruction tuning can be used in [llms](../llms/text-generation/README.md) microservice by assign its path to the environment variable `your_hf_llm_model`. ## 🚀4. Descriptions for Finetuning parameters From afe7d0f882a8b03837d5c276c9767ac919290ed2 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 21:51:38 +0800 Subject: [PATCH 16/29] test. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index b91734597..590329bbd 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -222,6 +222,7 @@ jobs: check_path=${{github.workspace}}/$(dirname "$refer_path")/$png_path fi real_path=$(realpath $check_path) + echo "$real_path" if [ $? -ne 0 ]; then echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" fail="TRUE" @@ -229,7 +230,6 @@ jobs: url=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps/||') response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") if [ "$response" -ne 200 ]; then - echo "$url" echo "**********Validation failed, try again**********" response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") if [ "$response_retry" -eq 200 ]; then From c433bfb283487e113228c405c812dd3fdeb6da69 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 21:57:35 +0800 Subject: [PATCH 17/29] Fix issue and add output Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 590329bbd..d21b0fad6 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -216,19 +216,19 @@ jobs: for png_line in $png_lines; do refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-) png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1) - if [[ "${png_path:0:1}" == "/" ]]; then + if [[ "${png_path:0:1}" == "/" || "${png_path:0:1}" == "#" ]]; then check_path=${{github.workspace}}$png_path else check_path=${{github.workspace}}/$(dirname "$refer_path")/$png_path fi real_path=$(realpath $check_path) - echo "$real_path" if [ $? -ne 0 ]; then echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" fail="TRUE" else url=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps/||') response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") + echo "$url" if [ "$response" -ne 200 ]; then echo "**********Validation failed, try again**********" response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") From d496b441ae5fe926943931bcc2b14a0856062d52 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 22:10:05 +0800 Subject: [PATCH 18/29] Fix issue and test Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 11 ++++++++++- LEGAL_INFORMATION.md | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index d21b0fad6..052e53ea3 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -216,11 +216,20 @@ jobs: for png_line in $png_lines; do refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-) png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1) - if [[ "${png_path:0:1}" == "/" || "${png_path:0:1}" == "#" ]]; then + if [[ "${png_path:0:1}" == "/" ]]; then check_path=${{github.workspace}}$png_path + elif [[ "${png_path:0:1}" == "#" ]]; then + check_path=${{github.workspace}}/$refer_path$png_path else check_path=${{github.workspace}}/$(dirname "$refer_path")/$png_path fi + if [[ "${png_path:0:1}" == "/" ]]; then + check_path=$png_path + elif [[ "${png_path:0:1}" == "#" ]]; then + check_path=$refer_path$png_path + else + check_path=$(dirname "$refer_path")/$png_path + fi real_path=$(realpath $check_path) if [ $? -ne 0 ]; then echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" diff --git a/LEGAL_INFORMATION.md b/LEGAL_INFORMATION.md index d196eb3a0..8a96f901b 100644 --- a/LEGAL_INFORMATION.md +++ b/LEGAL_INFORMATION.md @@ -2,7 +2,7 @@ 1. [License](#license) 2. [Citation](#citation) -3. [test](#test) +3. [test](/test) ## License From 8b7e98e58108c06a95917452c9f21e0679efdb9d Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 22:36:44 +0800 Subject: [PATCH 19/29] Add PR's own detection. Signed-off-by: ZePan110 --- .../workflows/pr-dockerfile-path-scan.yaml | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 052e53ea3..8310f97fb 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -210,7 +210,8 @@ jobs: cd ${{github.workspace}} fail="FALSE" branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}" - link_head="https://github.com/opea-project/GenAIComps/blob/main/" + echo $branch + link_head="https://github.com/opea-project/GenAIComps/blob/main" png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http') if [ -n "$png_lines" ]; then for png_line in $png_lines; do @@ -223,19 +224,12 @@ jobs: else check_path=${{github.workspace}}/$(dirname "$refer_path")/$png_path fi - if [[ "${png_path:0:1}" == "/" ]]; then - check_path=$png_path - elif [[ "${png_path:0:1}" == "#" ]]; then - check_path=$refer_path$png_path - else - check_path=$(dirname "$refer_path")/$png_path - fi real_path=$(realpath $check_path) if [ $? -ne 0 ]; then echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" fail="TRUE" else - url=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps/||') + url=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps||') response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") echo "$url" if [ "$response" -ne 200 ]; then @@ -244,8 +238,22 @@ jobs: if [ "$response_retry" -eq 200 ]; then echo "*****Retry successfully*****" else - echo "Invalid link from $real_path: $url, $png_line" - fail="TRUE" + echo "Retry failed. Check branch ${{ github.event.pull_request.head.ref }}" + # fail="TRUE" + + url_dev=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps/||') + response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url_dev") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from $real_path: $url_dev, $png_line" + fail="TRUE" + fi + fi + fi fi fi From 6a1854defd20a08263146daf7f6d6c34deb12456 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 22:46:16 +0800 Subject: [PATCH 20/29] reduce output Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 8310f97fb..755c4ea51 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -210,7 +210,6 @@ jobs: cd ${{github.workspace}} fail="FALSE" branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}" - echo $branch link_head="https://github.com/opea-project/GenAIComps/blob/main" png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http') if [ -n "$png_lines" ]; then @@ -231,7 +230,6 @@ jobs: else url=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps||') response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") - echo "$url" if [ "$response" -ne 200 ]; then echo "**********Validation failed, try again**********" response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") @@ -241,7 +239,8 @@ jobs: echo "Retry failed. Check branch ${{ github.event.pull_request.head.ref }}" # fail="TRUE" - url_dev=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps/||') + url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIComps||') + echo $url_dev response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev") if [ "$response" -ne 200 ]; then echo "**********Validation failed, try again**********" @@ -249,7 +248,7 @@ jobs: if [ "$response_retry" -eq 200 ]; then echo "*****Retry successfully*****" else - echo "Invalid link from $real_path: $url_dev, $png_line" + echo "Invalid link from $real_path: $url_dev" fail="TRUE" fi fi From f1415a435fe4aef499c3a5c4d69eb9864802db09 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 18 Sep 2024 22:55:50 +0800 Subject: [PATCH 21/29] Remove debug code. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 755c4ea51..22df7619b 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -237,10 +237,7 @@ jobs: echo "*****Retry successfully*****" else echo "Retry failed. Check branch ${{ github.event.pull_request.head.ref }}" - # fail="TRUE" - url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIComps||') - echo $url_dev response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev") if [ "$response" -ne 200 ]; then echo "**********Validation failed, try again**********" @@ -251,8 +248,9 @@ jobs: echo "Invalid link from $real_path: $url_dev" fail="TRUE" fi + else + echo "Check branch ${{ github.event.pull_request.head.ref }} successfully." fi - fi fi fi From a1f699e844f043e629b9632e4c71d7e77e68b7d5 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 19 Sep 2024 00:45:45 +0800 Subject: [PATCH 22/29] test Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 22df7619b..a8d6e09ac 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -208,6 +208,7 @@ jobs: - name: Checking Relative Path Validity run: | cd ${{github.workspace}} + echo ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.repo.full_name }} fail="FALSE" branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}" link_head="https://github.com/opea-project/GenAIComps/blob/main" From 2febdd89acfb107d1955ea4c589d9e59129556f3 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 19 Sep 2024 01:00:47 +0800 Subject: [PATCH 23/29] test. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index a8d6e09ac..ab246243d 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -208,9 +208,13 @@ jobs: - name: Checking Relative Path Validity run: | cd ${{github.workspace}} - echo ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.repo.full_name }} + echo ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.head.repo.full_name }} fail="FALSE" branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}" + letonghan:translation/manifest + https://github.com/opea-project/GenAIComps/blob/zepan/dev/test + https://github.com/letonghan/GenAIExamples/tree/translation/manifest + link_head="https://github.com/opea-project/GenAIComps/blob/main" png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http') if [ -n "$png_lines" ]; then From 0bffcfcdd112ae8fb54eb2cdc8aa2bcffb607b13 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 19 Sep 2024 17:02:50 +0800 Subject: [PATCH 24/29] Compatible with the origin of PR. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 22df7619b..672b2b8f9 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -209,7 +209,13 @@ jobs: run: | cd ${{github.workspace}} fail="FALSE" - branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}" + repo_name=${{ github.event.pull_request.head.repo.full_name }} + if [ "$(echo "$repo_name"|cut -d'/' -f1)" != "opea-project" ]; then + owner=$(echo "${{ github.event.pull_request.head.repo.full_name }}" |cut -d'/' -f1) + branch="https://github.com/$owner/GenAIComps/tree/${{ github.event.pull_request.head.ref }}" + else + branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}" + fi link_head="https://github.com/opea-project/GenAIComps/blob/main" png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http') if [ -n "$png_lines" ]; then From 9f1a77e8f9945a98c2e7a72828cdddc7231ce9fa Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 19 Sep 2024 17:18:15 +0800 Subject: [PATCH 25/29] Ignore links that require verification by a real person. Restore test files. Signed-off-by: ZePan110 --- .../workflows/pr-dockerfile-path-scan.yaml | 29 ++++++++++++------- LEGAL_INFORMATION.md | 1 - test | 1 - 3 files changed, 18 insertions(+), 13 deletions(-) delete mode 100644 test diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 672b2b8f9..3719c0fe4 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -168,24 +168,31 @@ jobs: - name: Check the Validity of Hyperlinks run: | + ignore_links=("https://platform.openai.com/docs/api-reference/fine-tuning" + "https://platform.openai.com/docs/api-reference/" + "https://openai.com/index/whisper/") cd ${{github.workspace}} fail="FALSE" url_lines=$(grep -Eo '\]\(http[s]?://[^)]+\)' --include='*.md' -r .) if [ -n "$url_lines" ]; then for url_line in $url_lines; do url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//') - path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) - response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") - if [ "$response" -ne 200 ]; then - echo "**********Validation failed, try again**********" - response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") - if [ "$response_retry" -eq 200 ]; then - echo "*****Retry successfully*****" - else - echo "Invalid link from ${{github.workspace}}/$path: $url" - fail="TRUE" + for ignore_link in "${ignore_links[@]}"; do + if [[ "$ignore_link" != "$url" ]]; then + path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) + response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from ${{github.workspace}}/$path: $url" + fail="TRUE" + fi + fi fi - fi + done done fi diff --git a/LEGAL_INFORMATION.md b/LEGAL_INFORMATION.md index 8a96f901b..7c71af2b9 100644 --- a/LEGAL_INFORMATION.md +++ b/LEGAL_INFORMATION.md @@ -2,7 +2,6 @@ 1. [License](#license) 2. [Citation](#citation) -3. [test](/test) ## License diff --git a/test b/test deleted file mode 100644 index 434d4406f..000000000 --- a/test +++ /dev/null @@ -1 +0,0 @@ -ttttttttttttttttttttttt \ No newline at end of file From fc54a05e170ee067de29b1d03db8922c3aafeb4a Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 19 Sep 2024 17:38:19 +0800 Subject: [PATCH 26/29] Change the judgment method. Signed-off-by: ZePan110 --- .../workflows/pr-dockerfile-path-scan.yaml | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index c92d474d3..22f5db7e0 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -168,31 +168,31 @@ jobs: - name: Check the Validity of Hyperlinks run: | - ignore_links=("https://platform.openai.com/docs/api-reference/fine-tuning" - "https://platform.openai.com/docs/api-reference/" - "https://openai.com/index/whisper/") + # ignore_links=("https://platform.openai.com/docs/api-reference/fine-tuning" + # "https://platform.openai.com/docs/api-reference/" + # "https://openai.com/index/whisper/") cd ${{github.workspace}} fail="FALSE" url_lines=$(grep -Eo '\]\(http[s]?://[^)]+\)' --include='*.md' -r .) if [ -n "$url_lines" ]; then for url_line in $url_lines; do url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//') - for ignore_link in "${ignore_links[@]}"; do - if [[ "$ignore_link" != "$url" ]]; then - path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) - response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") - if [ "$response" -ne 200 ]; then - echo "**********Validation failed, try again**********" - response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") - if [ "$response_retry" -eq 200 ]; then - echo "*****Retry successfully*****" - else - echo "Invalid link from ${{github.workspace}}/$path: $url" - fail="TRUE" - fi + path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) + if [[ "https://platform.openai.com/docs/api-reference/fine-tuning" == "$url" || "https://platform.openai.com/docs/api-reference/" == "$url" || "https://openai.com/index/whisper/" == "$url" ]]; then + echo "Link "$url" from ${{github.workspace}}/$path need to be verified by a real person." + else + response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from ${{github.workspace}}/$path: $url" + fail="TRUE" fi fi - done + fi done fi From a5bae967ee3a21885858e09e99dac4c66305bcbf Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 19 Sep 2024 17:43:04 +0800 Subject: [PATCH 27/29] Add need ignore link. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 22f5db7e0..25faa2e2d 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -167,10 +167,11 @@ jobs: uses: actions/checkout@v4 - name: Check the Validity of Hyperlinks - run: | - # ignore_links=("https://platform.openai.com/docs/api-reference/fine-tuning" + # ignore_links=("https://platform.openai.com/docs/api-reference/fine-tuning" # "https://platform.openai.com/docs/api-reference/" - # "https://openai.com/index/whisper/") + # "https://openai.com/index/whisper/" + # "https://platform.openai.com/docs/api-reference/chat/create") + run: | cd ${{github.workspace}} fail="FALSE" url_lines=$(grep -Eo '\]\(http[s]?://[^)]+\)' --include='*.md' -r .) @@ -178,7 +179,7 @@ jobs: for url_line in $url_lines; do url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//') path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) - if [[ "https://platform.openai.com/docs/api-reference/fine-tuning" == "$url" || "https://platform.openai.com/docs/api-reference/" == "$url" || "https://openai.com/index/whisper/" == "$url" ]]; then + if [[ "https://platform.openai.com/docs/api-reference/fine-tuning" == "$url" || "https://platform.openai.com/docs/api-reference/" == "$url" || "https://openai.com/index/whisper/" == "$url" || "https://platform.openai.com/docs/api-reference/chat/create" == "$url" ]]; then echo "Link "$url" from ${{github.workspace}}/$path need to be verified by a real person." else response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") @@ -215,7 +216,6 @@ jobs: - name: Checking Relative Path Validity run: | cd ${{github.workspace}} - echo ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.head.repo.full_name }} fail="FALSE" repo_name=${{ github.event.pull_request.head.repo.full_name }} if [ "$(echo "$repo_name"|cut -d'/' -f1)" != "opea-project" ]; then From 790ac04a41df08351f5a31e3caa7fd0a6df928fa Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 19 Sep 2024 17:49:46 +0800 Subject: [PATCH 28/29] Change runs-on. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index 25faa2e2d..cbb2f14a8 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -158,7 +158,7 @@ jobs: fi check-the-validity-of-hyperlinks-in-README: - runs-on: self-hosted #ubuntu-latest + runs-on: ubuntu-latest steps: - name: Clean Up Working Directory run: sudo rm -rf ${{github.workspace}}/* @@ -205,7 +205,7 @@ jobs: shell: bash check-the-validity-of-relative-path: - runs-on: self-hosted #ubuntu-latest + runs-on: ubuntu-latest steps: - name: Clean up Working Directory run: sudo rm -rf ${{github.workspace}}/* From 4f6177cfb4db028fb17754884c546b2054edf227 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 19 Sep 2024 19:32:33 +0800 Subject: [PATCH 29/29] Redefine output. Signed-off-by: ZePan110 --- .github/workflows/pr-dockerfile-path-scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index cbb2f14a8..fddd0a4df 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -259,7 +259,7 @@ jobs: if [ "$response_retry" -eq 200 ]; then echo "*****Retry successfully*****" else - echo "Invalid link from $real_path: $url_dev" + echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path" fail="TRUE" fi else