feat(tests): Add integration test framework, goldens for 4 APIs [gapic-generator-python]

.github/workflows/tests.yaml

@@ -313,6 +313,34 @@ jobs:
       - name: Submit coverage data to codecov.
         run: codecov
         if: always()
+  integration:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install system dependencies.
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y curl pandoc unzip gcc
+      - name: Install Bazel
+        run: |
+          wget -q "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/$BAZEL_BINARY"
+          wget -q "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/$BAZEL_BINARY.sha256"
+          sha256sum -c "$BAZEL_BINARY.sha256"
+          sudo dpkg -i "$BAZEL_BINARY"
+        env:
+          BAZEL_VERSION: 3.5.0
+          BAZEL_BINARY: bazel_3.5.0-linux-x86_64.deb
+      - name: Integration Tests
+        run: bazel test tests/integration:asset tests/integration:credentials tests/integration:logging tests/integration:redis
+
   style-check:
     runs-on: ubuntu-latest
     steps:
@@ -330,4 +358,4 @@ jobs:
             python -m pip install autopep8
       - name: Check diff
         run: |
-            find gapic tests -name "*.py" | xargs autopep8 --diff --exit-code
+            find gapic tests -name "*.py" -not -path 'tests/integration/goldens/*' | xargs autopep8 --diff --exit-code

DEVELOPMENT.md

@@ -5,3 +5,7 @@
 - Execute unit tests by running one of the sessions prefixed with `unit-`
   - Example: `nox -s unit-3.8`
 - Lint sources by running `autopep8`.
+
+## Integration Tests
+- Running tests: `bazel test tests/integration:asset`. See the full list of targets in `tests/integration/BUILD.bazel`.
+- Updating golden files: `bazel run tests/integration:asset_update`

WORKSPACE

@@ -55,3 +55,11 @@ apple_rules_dependencies()
 load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies")
 
 apple_support_dependencies()
+
+load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
+
+switched_rules_by_language(
+    name = "com_google_googleapis_imports",
+    gapic = True,
+    grpc = True,
+)

repositories.bzl

@@ -61,6 +61,21 @@ def gapic_generator_python():
         urls = ["https://github.com/googleapis/gapic-generator/archive/03abac35ec0716c6f426ffc1532f9a62f1c9e6a2.zip"],
     )
 
+    _rules_gapic_version = "0.5.3"
+    _maybe(
+        http_archive,
+        name = "rules_gapic",
+        strip_prefix = "rules_gapic-%s" % _rules_gapic_version,
+        urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version],
+    )
+
+    _maybe(
+        http_archive,
+        name = "com_google_googleapis",
+        strip_prefix = "googleapis-51fe6432d4076a4c101f561967df4bf1f27818e1",
+        urls = ["https://github.com/googleapis/googleapis/archive/51fe6432d4076a4c101f561967df4bf1f27818e1.zip"],
+    )
+
 def gapic_generator_register_toolchains():
     native.register_toolchains(
         "@gapic_generator_python//:pandoc_toolchain_linux",

rules_python_gapic/py_gapic.bzl

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@com_google_api_codegen//rules_gapic:gapic.bzl", "proto_custom_library")
+load("@rules_gapic//:gapic.bzl", "proto_custom_library")
 
 def py_gapic_library(
         name,
@@ -34,7 +34,7 @@ def py_gapic_library(
 
     file_args = {}
     if grpc_service_config:
-        file_args[grpc_service_config] =  "retry-config"
+        file_args[grpc_service_config] = "retry-config"
 
     proto_custom_library(
         name = srcjar_target_name,

rules_python_gapic/py_gapic_pkg.bzl

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@com_google_api_codegen//rules_gapic:gapic_pkg.bzl", "construct_package_dir_paths")
+load("@rules_gapic//:gapic_pkg.bzl", "construct_package_dir_paths")
 
 def _py_gapic_src_pkg_impl(ctx):
     srcjar_srcs = []
@@ -66,5 +66,3 @@ def py_gapic_assembly_pkg(name, deps, assembly_name = None, **kwargs):
         package_dir = package_dir,
         **kwargs
     )
-
-

rules_python_gapic/test/integration_test.bzl

@@ -0,0 +1,152 @@
+def _diff_integration_goldens_impl(ctx):
+    # Extract the Python source files from the generated 3 srcjars from API bazel target,
+    # and put them in the temporary folder `codegen_tmp`.
+    # Compare the `codegen_tmp` with the goldens folder e.g `tests/integration/goldens/redis`
+    # and save the differences in output file `diff_output.txt`.
+
+    diff_output = ctx.outputs.diff_output
+    check_diff_script = ctx.outputs.check_diff_script
+    gapic_library = ctx.attr.gapic_library
+    srcs = ctx.files.srcs
+    api_name = ctx.attr.name
+
+    script = """
+    mkdir codegen_tmp
+    unzip {input_srcs} -d codegen_tmp
+    diff -r codegen_tmp $PWD/tests/integration/goldens/{api_name} > {diff_output}
+    exit 0  # Avoid a build failure.
+    """.format(
+        diff_output = diff_output.path,
+        input_srcs = gapic_library[DefaultInfo].files.to_list()[0].path,
+        api_name = api_name,
+    )
+    ctx.actions.run_shell(
+        inputs = srcs + [
+            gapic_library[DefaultInfo].files.to_list()[0],
+        ],
+        outputs = [diff_output],
+        command = script,
+    )
+
+    # Check the generated diff_output file, if it is empty, that means there is no difference
+    # between generated source code and goldens files, test should pass. If it is not empty, then
+    # test will fail by exiting 1.
+
+    check_diff_script_content = """
+    # This will not print diff_output to the console unless `--test_output=all` option
+    # is enabled, it only emits the comparison results to the test.log.
+    # We could not copy the diff_output.txt to the test.log ($XML_OUTPUT_FILE) because that
+    # file is not existing at the moment. It is generated once test is finished.
+    cat $PWD/tests/integration/{api_name}_diff_output.txt
+    if [ -s $PWD/tests/integration/{api_name}_diff_output.txt ]
+    then
+        exit 1
+    fi
+    """.format(
+        api_name = api_name,
+    )
+
+    ctx.actions.write(
+        output = check_diff_script,
+        content = check_diff_script_content,
+    )
+    runfiles = ctx.runfiles(files = [ctx.outputs.diff_output])
+    return [DefaultInfo(executable = check_diff_script, runfiles = runfiles)]
+
+diff_integration_goldens_test = rule(
+    attrs = {
+        "gapic_library": attr.label(),
+        "srcs": attr.label_list(
+            allow_files = True,
+            mandatory = True,
+        ),
+    },
+    outputs = {
+        "diff_output": "%{name}_diff_output.txt",
+        "check_diff_script": "%{name}_check_diff_script.sh",
+    },
+    implementation = _diff_integration_goldens_impl,
+    test = True,
+)
+
+def integration_test(name, target, data):
+    # Bazel target `py_gapic_library` will generate 1 source jar that holds the
+    # Gapic_library's python sources.
+    diff_integration_goldens_test(
+        name = name,
+        gapic_library = target,
+        srcs = data,
+    )
+
+def _overwrite_golden_impl(ctx):
+    # Extract the Java source files from the generated 3 srcjars from API bazel target,
+    # and put them in the temporary folder `codegen_tmp`, zip as `goldens_output_zip`.
+    # Overwrite the goldens folder e.g `tests/integration/goldens/redis` with the
+    # code generation in `goldens_output_zip`.
+
+    gapic_library = ctx.attr.gapic_library
+    srcs = ctx.files.srcs
+
+    # Convert the name of bazel rules e.g. `redis_update` to `redis`
+    # because we will need to overwrite the goldens files in `redis` folder.
+    api_name = "_".join(ctx.attr.name.split("_")[:-1])
+    goldens_output_zip = ctx.outputs.goldens_output_zip
+
+    script = """
+    mkdir codegen_tmp
+    unzip {input_srcs} -d codegen_tmp
+    cd codegen_tmp
+    zip -r ../{goldens_output_zip} .
+    """.format(
+        goldens_output_zip = goldens_output_zip.path,
+        input_srcs = gapic_library[DefaultInfo].files.to_list()[0].path,
+    )
+
+    ctx.actions.run_shell(
+        inputs = srcs + [
+            gapic_library[DefaultInfo].files.to_list()[0],
+        ],
+        outputs = [goldens_output_zip],
+        command = script,
+    )
+
+    # Overwrite the goldens.
+    golden_update_script_content = """
+    cd ${{BUILD_WORKSPACE_DIRECTORY}}
+    # Filename pattern-based removal is needed to preserve the BUILD.bazel file.
+    find tests/Integration/goldens/{api_name}/ -name \\*.py-type f -delete
+    find tests/Integration/goldens/{api_name}/ -name \\*.json -type f -delete
+    unzip -ao {goldens_output_zip} -d tests/integration/goldens/{api_name}
+    """.format(
+        goldens_output_zip = goldens_output_zip.path,
+        api_name = api_name,
+    )
+    ctx.actions.write(
+        output = ctx.outputs.golden_update_script,
+        content = golden_update_script_content,
+        is_executable = True,
+    )
+    return [DefaultInfo(executable = ctx.outputs.golden_update_script)]
+
+overwrite_golden = rule(
+    attrs = {
+        "gapic_library": attr.label(),
+        "srcs": attr.label_list(
+            allow_files = True,
+            mandatory = True,
+        ),
+    },
+    outputs = {
+        "goldens_output_zip": "%{name}.zip",
+        "golden_update_script": "%{name}.sh",
+    },
+    executable = True,
+    implementation = _overwrite_golden_impl,
+)
+
+def golden_update(name, target, data):
+    overwrite_golden(
+        name = name,
+        gapic_library = target,
+        srcs = data,
+    )

tests/integration/BUILD.bazel

@@ -0,0 +1,78 @@
+load(
+    "@gapic_generator_python//rules_python_gapic:py_gapic.bzl",
+    "py_gapic_library",
+)
+load(
+    "@gapic_generator_python//rules_python_gapic:py_gapic_pkg.bzl",
+    "py_gapic_assembly_pkg",
+)
+load(
+    "@gapic_generator_python//rules_python_gapic/test:integration_test.bzl",
+    "golden_update",
+    "integration_test",
+)
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+package(default_visibility = ["//visibility:public"])
+
+####################################################
+# Integration Test Rules
+#
+# Usage:
+#    Run tests: bazel test tests/integration:asset
+#    Update goldens: bazel run tests/integration:asset_update
+####################################################
+
+INTEGRATION_TEST_LIBRARIES = [
+    "asset",  # Basic case.
+    "credentials",  # Check that the capital name edge case is handled.
+    "logging",  # Java package remapping in gapic.yaml.
+    "redis",  # Has a gapic.yaml.
+]
+
+[integration_test(
+    name = lib_name,
+    data = ["//tests/integration/goldens/%s:goldens_files" % lib_name],
+    target = ":%s_py_gapic" % lib_name,
+) for lib_name in INTEGRATION_TEST_LIBRARIES]
+
+[golden_update(
+    name = "%s_update" % lib_name,
+    data = ["//tests/integration/goldens/%s:goldens_files" % lib_name],
+    target = ":%s_py_gapic" % lib_name,
+) for lib_name in INTEGRATION_TEST_LIBRARIES]
+
+####################################################
+# API Library Rules
+####################################################
+
+# Asset.
+py_gapic_library(
+    name = "asset_py_gapic",
+    srcs = ["@com_google_googleapis//google/cloud/asset/v1:asset_proto"],
+    grpc_service_config = "cloudasset_grpc_service_config.json",
+)
+
+# Credentials.
+py_gapic_library(
+    name = "credentials_py_gapic",
+    srcs = ["@com_google_googleapis//google/iam/credentials/v1:credentials_proto"],
+    grpc_service_config = "iamcredentials_grpc_service_config.json",
+)
+
+# Logging.
+py_gapic_library(
+    name = "logging_py_gapic",
+    srcs = ["@com_google_googleapis//google/logging/v2:logging_proto"],
+    grpc_service_config = "logging_grpc_service_config.json",
+    opt_args = [
+        "python-gapic-namespace=google.cloud",
+        "python-gapic-name=logging",
+    ],
+)
+
+py_gapic_library(
+    name = "redis_py_gapic",
+    srcs = ["@com_google_googleapis//google/cloud/redis/v1:redis_proto"],
+    grpc_service_config = "redis_grpc_service_config.json",
+)