improve connector dependency github action (#18480)

* Use more specific variable name

* Write affected sources and destinations to files

* Use Markdown template to format results

* Update workflow to edit existing comment instead of making a new one

* Check whether changed_files.txt is empty before proceeding

* move template to .github/

* Use different quotes for true

* Move logic about continuing back to after dependency report because of ignores

* Use correct filename in conditional

* Use if to avoid early exit

* Create folder for comment templates and use constant for file path

* Add handling for non-source non-destination info, pull out markdown listing

.github/comment_templates/connector_dependency_template.md

@@ -0,0 +1,25 @@
+<!--- this comment is for `report-connectors-dependency.yml` identification, do not remove -->
+
+NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to run corresponding integration tests:
+
+<details>
+<summary>
+Sources ({num_sources})
+
+</summary>
+
+{sources}
+
+</details>
+
+<details>
+<summary>
+Destinations ({num_destinations})
+
+</summary>
+
+{destinations}
+
+</details>
+
+{others}

.github/workflows/report-connectors-dependency.yml

@@ -11,27 +11,32 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0 # OR "2" -> To retrieve the preceding commit.
-
       - name: Extract git-diff changed files
         run: |
           git diff --name-only remotes/origin/master...HEAD -- airbyte-integrations/ > ./changed_files.txt
-        id: extract_changed_files
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
       - name: Create dependency report
-        run: |
-          output=$(python ./tools/bin/ci_check_dependency.py ./changed_files.txt)
-          output="${output//'%'/'%25'}"
-          output="${output//$'\n'/'%0A'}"
-          output="${output//$'\r'/'%0D'}" 
-          echo "changed_files_report=$output" >> $GITHUB_OUTPUT
         id: dependency_report
+        run: |
+          python ./tools/bin/ci_check_dependency.py ./changed_files.txt
+          if [[ -f comment_body.md ]]; then
+            echo "comment=true" >> $GITHUB_OUTPUT
+          fi
+      - name: Find existing comment for connector dependencies
+        if: steps.dependency_report.outputs.comment == 'true'
+        uses: peter-evans/find-comment@v2
+        id: find_comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: "report-connectors-dependency.yml"
       - name: Comment report in PR
-        if: steps.dependency_report.outputs.changed_files_report != ''
+        if: steps.dependency_report.outputs.comment == 'true'
         uses: peter-evans/create-or-update-comment@v2
         with:
           issue-number: ${{ github.event.pull_request.number }}
-          body: |
-            **NOTE** :warning: Changes in this PR affect the following connectors. Make sure to run corresponding integration tests:
-            ${{steps.dependency_report.outputs.changed_files_report}}
+          comment-id: ${{ steps.find_comment.outputs.comment-id }}
+          edit-mode: "replace"
+          body-file: "comment_body.md"

tools/bin/ci_check_dependency.py

@@ -11,6 +11,7 @@
     "/integration_tests/", "/unit_tests/",
     # Common
     "acceptance-test-config.yml", "acceptance-test-docker.sh", ".md", ".dockerignore", ".gitignore", "requirements.txt"]
+COMMENT_TEMPLATE_PATH = ".github/comment_templates/connector_dependency_template.md"
 
 
 def main():
@@ -32,7 +33,10 @@ def main():
 
     # Try to find dependency in build.gradle file
     depended_connectors = list(set(get_depended_connectors(changed_modules, all_build_gradle_files)))
-    write_report(depended_connectors)
+
+    # Create comment body to post on pull request
+    if depended_connectors:
+        write_report(depended_connectors)
 
 
 def get_changed_files(path):
@@ -73,19 +77,53 @@ def find_file(name, path):
             return os.path.join(root, name)
 
 
-def get_depended_connectors(changed_connectors, all_build_gradle_files):
+def get_depended_connectors(changed_modules, all_build_gradle_files):
     depended_connectors = []
-    for changed_connector in changed_connectors:
+    for changed_module in changed_modules:
         for connector, gradle_file in all_build_gradle_files.items():
             with open(gradle_file) as file:
-                if changed_connector in file.read():
+                if changed_module in file.read():
                     depended_connectors.append(connector)
     return depended_connectors
 
 
+def as_bulleted_markdown_list(items):
+    text = ""
+    for item in items:
+        text += f"- {item}\n"
+    return text
+
+
 def write_report(depended_connectors):
+    affected_sources = []
+    affected_destinations = []
+    affected_others = []
     for depended_connector in depended_connectors:
-        print("- " + depended_connector)
+        if depended_connector.startswith("source"):
+            affected_sources.append(depended_connector)
+        elif depended_connector.startswith("destination"):
+            affected_destinations.append(depended_connector)
+        else:
+            affected_others.append(depended_connector)
+
+    with open(COMMENT_TEMPLATE_PATH, "r") as f:
+        template = f.read()
+
+    others_md = ""
+    if affected_others:
+        others_md += "The following were also affected:\n"
+        others_md += as_bulleted_markdown_list(affected_others)
+
+    comment = template.format(
+        sources=as_bulleted_markdown_list(affected_sources),
+        destinations=as_bulleted_markdown_list(affected_destinations),
+        others=others_md,
+        num_sources=len(affected_sources),
+        num_destinations=len(affected_destinations)
+    )
+
+    with open("comment_body.md", "w") as f:
+        f.write(comment)
 
 
 if __name__ == "__main__":